From e7fd8777a285b9c7a5a44285e520a656dadc23fe Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 13 Nov 2026 18:01:26 +0800 Subject: [PATCH 1/6] refactor(context)!: separate table loading from execution settings Introduce TableLoadOptions for static factories and keep operation Contexts limited to execution settings. Apply dynamic options through Table::Copy while keeping NewScan/NewRead/NewWrite/NewCommit context-only. Preserve catalog ownership, storage bindings, schema caches and query-auth behavior, and migrate callers, tests, examples and documentation. Refs #555 BREAKING CHANGE: Context builders no longer accept table loading parameters or option maps. Static factories take TableLoadOptions and dynamic options; loaded-table option overrides use Table::Copy. --- benchmark/benchmark_suite.cpp | 57 +- docs/source/api/catalog.rst | 3 + docs/source/api/format_table.rst | 22 +- docs/source/user_guide/catalog.rst | 266 +-- docs/source/user_guide/commit.rst | 2 +- docs/source/user_guide/file_index.rst | 4 +- docs/source/user_guide/format_table.rst | 42 +- docs/source/user_guide/global_index.rst | 33 +- docs/source/user_guide/manifest_cache.rst | 2 +- .../user_guide/manifest_entry_cache.rst | 12 +- .../user_guide/parquet_metadata_cache.rst | 4 +- docs/source/user_guide/read.rst | 5 +- docs/source/user_guide/realtime.rst | 29 +- docs/source/user_guide/write.rst | 4 +- examples/clean_demo.cpp | 24 +- examples/read_write_demo.cpp | 20 +- include/paimon/catalog/table.h | 9 +- include/paimon/catalog/table_load_options.h | 51 + include/paimon/commit_context.h | 136 +- include/paimon/file_store_commit.h | 13 +- include/paimon/file_store_write.h | 9 +- include/paimon/read_context.h | 172 +- include/paimon/scan_context.h | 131 +- include/paimon/table/format/format_table.h | 16 +- include/paimon/table/source/table_read.h | 7 +- include/paimon/table/source/table_scan.h | 7 +- include/paimon/write_context.h | 147 +- .../append_compact_coordinator_test.cpp | 21 +- src/paimon/core/catalog/catalog.cpp | 3 +- .../core/catalog/file_system_catalog.cpp | 19 +- src/paimon/core/catalog/file_system_catalog.h | 4 +- .../core/catalog/file_system_catalog_test.cpp | 18 +- .../global_index/global_index_write_task.cpp | 20 +- ...eted_primary_key_index_maintainer_test.cpp | 74 +- .../pksorted/pk_sorted_data_file_reader.cpp | 12 +- ..._changelog_merge_tree_compact_rewriter.cpp | 9 +- .../lookup_merge_tree_compact_rewriter.cpp | 9 +- ...erge_tree_compact_manager_factory_test.cpp | 10 +- .../compact/merge_tree_compact_rewriter.cpp | 9 +- .../merge_tree_compact_rewriter_test.cpp | 9 +- src/paimon/core/mergetree/lookup_levels.cpp | 9 +- .../append_only_file_store_scan_test.cpp | 47 +- .../append_only_file_store_write.cpp | 9 +- .../append_only_file_store_write_test.cpp | 111 +- ...eduplicate_compaction_changes_provider.cpp | 10 +- src/paimon/core/operation/commit_context.cpp | 160 +- .../core/operation/commit_context_test.cpp | 170 +- .../data_evolution_split_read_test.cpp | 94 +- .../operation/file_store_commit_impl_test.cpp | 1509 +++++++----- .../core/operation/file_store_commit_test.cpp | 57 +- .../core/operation/file_store_write_test.cpp | 171 +- .../core/operation/internal_read_context.cpp | 90 +- .../core/operation/internal_read_context.h | 28 +- .../operation/internal_read_context_test.cpp | 320 ++- .../key_value_file_store_write_test.cpp | 200 +- .../operation/merge_file_split_read_test.cpp | 329 ++- .../precise_file_index_read_test.cpp | 202 +- .../operation/raw_file_split_read_test.cpp | 219 +- src/paimon/core/operation/read_context.cpp | 200 +- .../core/operation/read_context_test.cpp | 324 +-- src/paimon/core/operation/scan_context.cpp | 191 +- .../core/operation/scan_context_test.cpp | 197 +- src/paimon/core/operation/write_context.cpp | 175 +- .../core/operation/write_context_test.cpp | 170 +- src/paimon/core/snapshot_file_scan_test.cpp | 31 +- src/paimon/core/table/file_store_table.cpp | 258 +-- src/paimon/core/table/file_store_table.h | 37 +- src/paimon/core/table/format/format_table.cpp | 61 +- .../core/table/format/format_table_test.cpp | 682 ++++-- .../core/table/sink/commit_message_test.cpp | 18 +- .../table/source/append_count_reader_test.cpp | 21 +- .../table/source/filtered_row_id_reader.cpp | 39 +- .../table/source/pk_count_reader_test.cpp | 25 +- .../table/source/query_auth_table_read.cpp | 30 +- .../core/table/source/query_auth_table_read.h | 8 +- .../source/query_auth_table_read_test.cpp | 14 +- .../table/source/query_auth_table_scan.cpp | 47 +- .../core/table/source/query_auth_table_scan.h | 11 +- .../source/scan_exception_safety_test.cpp | 20 +- .../core/table/source/table_read_test.cpp | 238 +- .../core/table/source/table_scan_test.cpp | 78 +- .../table/system/audit_log_system_table.cpp | 44 +- .../table/system/audit_log_system_table.h | 6 +- .../core/table/system/binlog_system_table.cpp | 13 +- .../core/table/system/binlog_system_table.h | 3 + .../table/system/global_system_tables.cpp | 24 + .../core/table/system/global_system_tables.h | 12 + .../table/system/in_memory_system_table.cpp | 7 +- .../table/system/metadata_system_tables.cpp | 66 +- .../table/system/metadata_system_tables.h | 24 + .../system/read_optimized_system_table.cpp | 36 +- .../system/read_optimized_system_table.h | 3 + src/paimon/core/table/system/system_table.cpp | 32 +- src/paimon/core/table/system/system_table.h | 8 +- .../core/table/system/system_table_test.cpp | 10 +- src/paimon/core/table/table_factory.cpp | 528 ++--- src/paimon/core/table/table_factory.h | 19 +- src/paimon/core/table/table_test.cpp | 1830 ++++++++++----- src/paimon/core/utils/blob_view_lookup.cpp | 30 +- src/paimon/rest/rest_catalog.cpp | 6 +- src/paimon/rest/rest_catalog_test.cpp | 251 +- src/paimon/testing/mock/mock_catalog.h | 15 + src/paimon/testing/utils/test_helper.h | 74 +- test/inte/append_compaction_inte_test.cpp | 25 +- test/inte/blob_table_inte_test.cpp | 61 +- test/inte/clean_inte_test.cpp | 208 +- test/inte/data_evolution_table_test.cpp | 194 +- test/inte/global_index_test.cpp | 77 +- test/inte/nested_column_pruning_inte_test.cpp | 118 +- test/inte/paimon_read_compat_inte_test.cpp | 27 +- test/inte/pk_compaction_inte_test.cpp | 184 +- .../primary_key_sorted_index_inte_test.cpp | 42 +- test/inte/read_inte_test.cpp | 866 ++++--- test/inte/realtime_write_inte_test.cpp | 595 +++-- test/inte/scan_and_read_inte_test.cpp | 2062 +++++++++++------ test/inte/scan_inte_test.cpp | 740 ++++-- test/inte/write_and_read_inte_test.cpp | 369 ++- .../write_and_read_inte_with_index_test.cpp | 108 +- test/inte/write_inte_test.cpp | 445 ++-- 119 files changed, 9925 insertions(+), 7291 deletions(-) create mode 100644 include/paimon/catalog/table_load_options.h diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp index 0a48a9173..307ee6abc 100644 --- a/benchmark/benchmark_suite.cpp +++ b/benchmark/benchmark_suite.cpp @@ -37,6 +37,7 @@ #include "benchmark/cli_option_parsing.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/testing/utils/testharness.h" #if __has_include("parquet/arrow/reader.h") @@ -434,25 +435,33 @@ Status WriteSourceDataToWriter(paimon::FileStoreWrite* writer, const SourceDataS Status WriteAndCommit(const std::string& table_path, const std::map& options, const SourceDataSpec& source_spec) { - paimon::WriteContextBuilder write_builder(table_path, "benchmark-writer"); + paimon::TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + paimon::WriteContextBuilder write_builder("benchmark-writer"); + write_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_ctx, + AddContext(write_builder.Finish(), "create write context")); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr write_ctx, - AddContext(write_builder.SetOptions(options).Finish(), "create write context")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - AddContext(paimon::FileStoreWrite::Create(std::move(write_ctx)), - "create file store writer")); + std::unique_ptr writer, + AddContext(paimon::FileStoreWrite::Create(write_builder_load_options, std::move(write_ctx), + write_builder_options), + "create file store writer")); PAIMON_RETURN_NOT_OK(WriteSourceDataToWriter(writer.get(), source_spec)); PAIMON_ASSIGN_OR_RAISE(std::vector> messages, AddContext(writer->PrepareCommit(), "prepare commit")); - paimon::CommitContextBuilder commit_builder(table_path, "benchmark-writer"); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_ctx, - AddContext(commit_builder.SetOptions(options).Finish(), "create commit context")); + paimon::TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + paimon::CommitContextBuilder commit_builder("benchmark-writer"); + commit_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_ctx, + AddContext(commit_builder.Finish(), "create commit context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr committer, - AddContext(paimon::FileStoreCommit::Create(std::move(commit_ctx)), "create committer")); + AddContext(paimon::FileStoreCommit::Create(commit_builder_load_options, + std::move(commit_ctx), commit_builder_options), + "create committer")); PAIMON_RETURN_NOT_OK(AddContext(committer->Commit(messages), "commit write")); return Status::OK(); } @@ -562,20 +571,26 @@ Result GetOrCreateSharedReadTable(const std::string Result ReadRows(const std::string& table_path, const std::map& options, int32_t prefetch_parallel_num) { - paimon::ScanContextBuilder scan_builder(table_path); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan_ctx, - AddContext(scan_builder.SetOptions(options).Finish(), "create scan context")); + paimon::TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + paimon::ScanContextBuilder scan_builder; + scan_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_ctx, + AddContext(scan_builder.Finish(), "create scan context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scanner, - AddContext(paimon::TableScan::Create(std::move(scan_ctx)), "create scanner")); + AddContext(paimon::TableScan::Create(scan_builder_load_options, std::move(scan_ctx), + scan_builder_options), + "create scanner")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, AddContext(scanner->CreatePlan(), "create plan")); - paimon::ReadContextBuilder read_builder(table_path); + paimon::TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + paimon::ReadContextBuilder read_builder; constexpr int32_t kPrefetchBatchCount = 600; - read_builder.SetOptions(options) - .EnablePrefetch(true) + read_builder_options = options; + read_builder.EnablePrefetch(true) .SetPrefetchBatchCount(kPrefetchBatchCount) .SetPrefetchMaxParallelNum(prefetch_parallel_num) .EnableMultiThreadRowToBatch(GetRowToBatchThreadNumber() > 1) @@ -584,7 +599,9 @@ Result ReadRows(const std::string& table_path, AddContext(read_builder.Finish(), "create read context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr reader, - AddContext(paimon::TableRead::Create(std::move(read_ctx)), "create table reader")); + AddContext(paimon::TableRead::Create(read_builder_load_options, std::move(read_ctx), + read_builder_options), + "create table reader")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, AddContext(reader->CreateReader(plan->Splits()), "create batch reader")); diff --git a/docs/source/api/catalog.rst b/docs/source/api/catalog.rst index df247f4a8..df19a17c6 100644 --- a/docs/source/api/catalog.rst +++ b/docs/source/api/catalog.rst @@ -37,5 +37,8 @@ Interface .. doxygenclass:: paimon::Table :members: +.. doxygenstruct:: paimon::TableLoadOptions + :members: + .. doxygenstruct:: paimon::CatalogOptions :members: diff --git a/docs/source/api/format_table.rst b/docs/source/api/format_table.rst index 1ec964375..7464205f7 100644 --- a/docs/source/api/format_table.rst +++ b/docs/source/api/format_table.rst @@ -35,18 +35,10 @@ and :cpp:class:`paimon::FileStoreCommit`. See :doc:`../user_guide/format_table`. Contexts ======== -Each of those entry points builds its context from a table path, and the schema under that path -says what kind of table it is. A caller that already holds a ``FormatTable`` hands it over instead, -through the constructor each context builder has for one: - -* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::shared_ptr&)`` - -This is the only way to reach a format table whose schema does not live under its own location, -such as one a REST catalog serves: nothing under the location says that it is a format table, nor -that what sits below it is data rather than metadata. The table knows both, so a setting that would -answer either question again - ``SetTableSchema()``, ``WithFileSystem()`` and a branch among them - -is refused by these builders rather than quietly ignored. Options given at the call still win over -the ones the table carries, as they do everywhere else. +Static entry points load the table through :cpp:struct:`paimon::TableLoadOptions`. +Operation Contexts provide execution settings; dynamic options are a separate +argument. A caller holding a loaded ``FormatTable`` creates operators directly +with its ``NewScan()``, ``NewRead()``, ``NewWrite()`` or ``NewCommit()`` methods. +This also works when a catalog keeps the schema outside the table location. +The loaded table supplies its schema and file system, and dynamic options passed +to ``New*()`` override the stored options. diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index d9ee91f66..9a3aaabd6 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -121,66 +121,65 @@ through the regular ``Catalog`` API, and table snapshots can be listed through Committing through the catalog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A REST catalog reports ``Catalog::SupportsVersionManagement() == true`` and -publishes snapshots through the server. Pass the catalog and table identifier -to ``CommitContextBuilder::WithCatalog`` to commit on local or object storage -without an atomic snapshot-file rename: +publishes snapshots through the server. A loaded table retains its catalog +services and UUID. Writers and committers created from that table use the same +identity and per-table credentials: .. code-block:: cpp - // Writers and committers share ownership of the catalog. std::shared_ptr shared_catalog(std::move(catalog)); - - // Load the table's identity before preparing changes. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); - - paimon::CommitContextBuilder builder(table_path, "commit-user"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); - // CatalogUuid() stays null when the server supplies no id. - if (std::optional table_uuid = table->CatalogUuid()) { - builder.WithTableId(table_uuid.value()); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, + paimon::WriteContextBuilder("commit-user").Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + table->NewWrite(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + paimon::CommitContextBuilder("commit-user").Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - paimon::FileStoreCommit::Create(std::move(context))); + table->NewCommit(std::move(commit_context))); PAIMON_RETURN_NOT_OK(commit->Commit(commit_messages)); -Configure native writers with the same catalog and identifier. They load the -current schema and latest snapshot from the catalog, including committed -offsets when a real-time writer is created: +``Catalog::GetTable()`` returns a common ``Table`` interface for managed, format +and system tables. Managed tables select append-only or primary-key behavior +from the loaded schema; system tables reject writes and commits. +Loading a managed table's schema, location and catalog UUID does not require +its data-format plugin; the plugin is resolved when creating an operator that +uses the format. + +``NewCommit()``, ``NewWrite()``, ``NewRead()`` and ``NewScan()`` take an operation +Context containing execution settings. Apply dynamic options with ``Table::Copy()`` +before creating operators; the original table remains unchanged. Operators use +the copy's bound type, path, schema, options, file system and catalog services. +``GetSchema()`` returns the +bound schema without metadata I/O; reload through the catalog to refresh it. +Branch or type options that contradict a loaded managed table are rejected. +See the :ref:`Table API `. + +The static factories instead take ``TableLoadOptions``, an operation Context and +an optional dynamic options map. Loading parameters are independent of the +Context; finishing a Context performs no table metadata I/O: .. code-block:: cpp - paimon::WriteContextBuilder write_builder(table_path, "commit-user"); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - paimon::FileStoreWrite::Create(std::move(write_context))); - -``Catalog::GetTable()`` returns a common ``Table`` interface for managed, format and -system tables. The managed implementation selects append-only or primary-key behavior -from the loaded schema; system tables reject writes and commits. Loading a managed -table's schema, location and catalog UUID does not require its data-format plugin; -the plugin is resolved when creating an operator that uses the format. - -``table->NewCommit()``, ``NewWrite()``, ``NewRead()`` and ``NewScan()`` use the table's -bound type, path, schema and file system. ``GetSchema()`` returns that schema without -metadata I/O; reload through the catalog to refresh it. Contexts still provide execution -settings, dynamic options and optional catalog services. Branch or type options that -contradict a loaded managed table are rejected. The legacy static ``Create()`` methods -load the table from their context before delegating to these methods. -See the :ref:`Table API `. + paimon::TableLoadOptions load_options; + load_options.catalog = shared_catalog; + load_options.identifier.emplace("db", "tbl"); + // Optional expected UUID, validated against the catalog during loading. + load_options.table_id = table->CatalogUuid(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + paimon::CommitContextBuilder("commit-user").Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + paimon::FileStoreCommit::Create(load_options, std::move(context))); -An explicit ``WithBranch()`` or branch-qualified identifier participates in the same -branch check. Omitting a branch allows the loaded table to supply it. The static -scan and read factories use an explicit context branch first, then the branch in -the schema options, and finally main. +For file-system loading, set ``TableLoadOptions::path``. Optional fields specify +an explicit file system, URI scheme routing or a serialized Paimon table schema. +The serialized table schema is separate from the Arrow projection in +``ReadContextBuilder::SetReadSchema()``. Set the branch through the identifier +or the dynamic ``branch`` option; conflicting values are rejected. -Global system tables under ``sys`` retain their catalog for deferred row loading. -Before calling ``GetTable()`` for one, place the catalog in a ``std::shared_ptr``, -as in the example above. ``LoadTableSchema()`` can inspect the schema without -retaining the catalog. +Global system tables under ``sys`` retain their catalog for deferred row +loading. Keep the catalog in a ``std::shared_ptr`` before loading these tables. For real-time writes, also set ``WithStreamingMode(true)`` and ``WithRealtimeContext(...)`` with ``realtime.enabled=true``. After @@ -191,14 +190,12 @@ already advanced the latest snapshot, the requested snapshot must be published under the table path. Recreating the writer restores offsets from the catalog's current snapshot. -Both builders use the file system of the table being written or committed for -manifests and data, so a catalog that issues temporary credentials per table -serves them through ``Catalog::GetTableFileSystem``. ``WithFileSystem`` overrides -it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override -file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)`` -instead. +Table loading uses ``Catalog::GetTableFileSystem`` for per-table credentials. +An explicit ``TableLoadOptions::file_system`` or +``fs_scheme_to_identifier_map`` controls file-system resolution instead. +Loaded format tables create operators directly with their ``New*()`` methods. -A file system passed to ``WithFileSystem`` is used exactly as it is: neither the +A file system passed in ``TableLoadOptions::file_system`` is used exactly as it is: neither the built-in file systems nor the table's own credentials are involved, so a custom one has to authenticate its own accesses. Draw credentials that expire from a ``CredentialProvider`` you build with ``CredentialProviderFactory``: hold the @@ -212,8 +209,9 @@ returns credentials that are still valid and reloads them before they expire. "oss://bucket/tbl", options)); // MyFileSystem merges provider->GetCredentials() over its own options per access. auto my_fs = std::make_shared(provider); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) - .WithFileSystem(my_fs); + load_options.catalog = shared_catalog; + load_options.identifier.emplace("db", "tbl"); + load_options.file_system = my_fs; The snapshot being committed carries a uuid generated on the client, and the commit names the snapshot it is based on by that snapshot's uuid, so the server @@ -224,12 +222,12 @@ and ``commit.timeout`` as a file-system commit is. A REST catalog can also give a writer a commit lease, so concurrent writers take turns instead of racing. The table property ``rest.commit.lock-enabled = true`` turns on lease enforcement on the server, and a writer that sets -``rest.commit.acquire-lock = true``, usually as a commit context option, acquires +``rest.commit.acquire-lock = true``, as a dynamic commit option, acquires a lease for each publication attempt. Both default to ``false``; enabling the table property alone does not make writers acquire, and asking to acquire without it fails when the commit is created. A writer with an active lease keeps the others out, including writers that do not acquire one. The lease is bound to the -table or branch, to the table UUID, which has to be passed to ``WithTableId``, +table or branch, to the table UUID retained by the loaded table, and to the exact commit user, so give each logical writer its own commit user and keep it across recovery. @@ -241,10 +239,13 @@ keep it across recovery. if (!table_uuid) { return paimon::Status::Invalid("a commit lease needs the table UUID"); } - paimon::CommitContextBuilder builder(table_path, "writer-1"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) - .WithTableId(table_uuid.value()) - .AddOption(paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + paimon::CommitContextBuilder("writer-1").Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr commit_table, + table->Copy( + {{paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"}})); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + commit_table->NewCommit(std::move(context))); Each attempt acquires its own lease before reading the latest snapshot, waits while another writer holds it until ``commit.timeout`` without using up @@ -269,7 +270,7 @@ under it. Ending an attempt that published nothing only stops renewing the lease, without waiting for a renewal in flight, and the lease stays held until it expires. -Leases need ``WithCatalog``: a commit that only builds the commit table request +Leases need a table loaded through a catalog: a commit that only builds the commit table request for the caller to send cannot hold one, so asking to acquire is rejected there. The snapshot UUID checks above still apply. @@ -287,8 +288,9 @@ A catalog response of ``{"snapshot": null}`` means the table has no snapshot. Only a catalog reporting that snapshot loading is unsupported falls back to file-system lookup; other catalog errors propagate to the caller. -Read ``Table::CatalogUuid()`` before preparing changes and pass it to -``WithTableId`` so the server can reject commits to a dropped and recreated table. +Load the table before preparing changes so its retained ``CatalogUuid()`` lets +the server reject commits to a dropped and recreated table. Static commit +loading accepts the expected UUID through ``TableLoadOptions::table_id``. ``Table::Uuid()`` can fall back to the table name and must not supply this id. An unset table id is sent as null and subject to server validation. Each commit attempt to the main branch reloads the catalog's current schema id for the new @@ -336,57 +338,35 @@ so its restored file references are visible to the expiration operation. Reading through the catalog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To read a native table through its catalog, pass the catalog and table -identifier to ``ReadContextBuilder::WithCatalog`` or -``ScanContextBuilder::WithCatalog``. The catalog then supplies: - -* the table's file system, with the per-table temporary credentials the catalog - issues, unless ``WithFileSystem`` sets one; -* whether the table enables ``query-auth.enabled``, read from the schema the - catalog holds for the branch read; -* for a table that does, its row filters and column masks. - -Passing only the file system through ``WithFileSystem`` gives neither of the -last two. - -The fields a scan plans with and a read reads with do not come from the catalog, -but from the schema ``SetTableSchema`` hands in for the main branch or, otherwise, -from the latest schema published under the table path of the branch read. That -schema has to exist even when the catalog holds the current one: a table whose -schema only the catalog holds cannot be scanned or read this way yet, and when -the two differ, the fields are those of the schema under the table path. A -query-auth scan also checks column identities and rule types against the -catalog's schema on every plan: +Load the table through a shared catalog and create read and scan operators from it: .. code-block:: cpp - // Readers and scanners share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - - paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_builder.Finish()); + paimon::ScanContextBuilder().Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); - - paimon::ReadContextBuilder read_builder(table_path); - read_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, - read_builder.Finish()); + paimon::ReadContextBuilder().Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - paimon::TableRead::Create(std::move(read_context))); - -When ``Finish()`` builds the context, it asks the catalog for the table's file -system through ``Catalog::GetTableFileSystem`` and uses it as-is for the schema, -snapshots, manifests and data under the table path, so it signs every access with -the table's credentials and reloads them as they expire. An explicit -``WithFileSystem`` takes precedence, so the catalog is not asked for one. A -catalog that issues no per-table credentials returns its catalog-level file -system, so this is also how you read with the catalog's own object-store -credentials. For a format table, use ``ReadContextBuilder(FormatTable)`` or -``ScanContextBuilder(FormatTable)`` instead; ``WithCatalog`` is rejected because -the table already carries the file system it was loaded through. + table->NewRead(std::move(read_context))); + +Context builders retain execution settings only; ``Finish()`` performs no table +metadata I/O. The loaded table retains its schema, location, identifier, catalog +services and per-table file system. That file system signs metadata and data +accesses with the table's credentials and refreshes them as they expire. + +Static factories take ``TableLoadOptions`` separately. Its explicit ``file_system`` +takes precedence over the catalog's file system. Without one, the catalog supplies +its per-table credentials, or its catalog-level file system when it issues none. +Use ``NewScan`` and ``NewRead`` on a loaded format table too; its storage binding +and external-directory semantics remain those of the loaded table. + +Dependency reads retain the catalog configuration and URI routing. REST dependency +reads preserve an existing ``X-Paimon-Read-Via`` header or derive it from the outer +table identifier. Row filters and column masks ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -396,11 +376,11 @@ any of these sets it: * the latest schema under the table path; * the options of the scan or read; -* for a scan or read built ``WithCatalog``, the schema the catalog holds for the +* for a scan or read of a table loaded through its catalog, the schema the catalog holds for the branch read, which ``TableScan::Create`` and ``TableRead::Create`` load as Java loads the table of that branch. -A schema handed in with ``SetTableSchema`` for the main branch does not decide it +A schema handed in through ``TableLoadOptions::table_schema_json`` for the main branch does not decide it alone: unless the catalog already enabled the option, the latest schema under the table path is checked too, so a cache taken before the option was turned on still applies the rules. @@ -466,7 +446,7 @@ Every plan of such a table, the first included, asks the catalog for the rules of the query (``POST /v1/{prefix}/databases/{database}/tables/{table}/auth``), checks that the columns they name still exist with the types they were written for and are columns of the schema the scan plans with, and attaches them to each -split. A scan therefore has to be built ``WithCatalog``, and a catalog that does +split. A scan therefore has to use a table loaded through its catalog, and a catalog that does not support the query auth interface is refused with ``NotImplemented``. The scan path must be the location ``Catalog::GetTableLocation`` returns for the table and branch the rules are asked for, so the rules of one table are never @@ -478,13 +458,13 @@ system table path such as ``tbl$branch_dev$files`` takes it from the branch it names. An identifier in the ``sys`` database can only be used with a global system table path (``/sys/``); it cannot skip the catalog policy of a data table or its per-table system tables. -The branch the identifier, the ``branch`` option, ``WithBranch()`` or a -system table path names must agree, which ``Finish()`` checks before it asks the +The branch the identifier, the dynamic ``branch`` option or a +system table path names must agree, which the static factory checks before it asks the catalog for anything. A branch named only by the option or the path is carried into the identifier before requesting the table's file system, so the ``query-auth.enabled`` setting, the credentials and the rules all come from the branch the scan plans. -``Finish()`` also refuses, as for a write, a branch the catalog would read as +The static factory also refuses, as for a write, a branch the catalog would read as another object, such as ``MAIN`` or a name holding a ``$`` (see `Naming the branch`_). The request selects every column unless ``ScanContextBuilder::SetReadFieldNames`` names the columns the query reads; the @@ -518,7 +498,7 @@ the rules a split carries and refuses a split planned without them. It refuses a split authorized for another table location or branch, even when their schemas and data file names match. This check also applies to a worker without the catalog. The read only has to know that the table enables query auth, which the -schema under the table path, ``WithCatalog`` or the read option +schema under the table path, the bound catalog or the dynamic read option ``query-auth.enabled = true`` tells it. It refuses a split when a column it outputs or filters on is not among those the split was authorized for or is now another field under the same name, as after a rename or a column added since the @@ -571,54 +551,59 @@ These examples use ``shared_catalog`` from `Reading through the catalog`_. using namespace paimon; // Plan with the columns and the filter the read uses. - ScanContextBuilder scan_builder(table_path); - scan_builder.WithCatalog(shared_catalog, Identifier("db", "tbl")) - .SetReadFieldNames({"name", "phone"}) + TableLoadOptions scan_load{table_path}; + scan_load.catalog = shared_catalog; + scan_load.identifier.emplace("db", "tbl"); + ScanContextBuilder scan_builder; + scan_builder.SetReadFieldNames({"name", "phone"}) .SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_load, std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); // A worker without the catalog tells the read that the table enables query auth. - ReadContextBuilder read_builder(table_path); - read_builder.AddOption(Options::QUERY_AUTH_ENABLED, "true") - .SetReadFieldNames({"name", "phone"}) + TableLoadOptions read_load{table_path}; + ReadContextBuilder read_builder; + read_builder.SetReadFieldNames({"name", "phone"}) .SetPredicate(predicate) .EnablePredicateFilter(true); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_load, std::move(read_context), + {{Options::QUERY_AUTH_ENABLED, "true"}})); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); The read does not take the branch from the splits or from ``query-auth.enabled``: a worker reading a branch names it itself, as the scan -did. As for a scan, a branch named by the ``branch`` option or ``WithBranch`` is -carried into the identifier of a read built ``WithCatalog``, so the catalog +did. As for a scan, a branch named by the dynamic ``branch`` option is +carried into the identifier supplied in ``TableLoadOptions``, so the catalog hands out the credentials of that branch. .. code-block:: cpp // Plan branch "dev"; Identifier("db", "tbl", "dev") names it as well. - ScanContextBuilder dev_scan_builder(table_path); - dev_scan_builder.WithCatalog(shared_catalog, Identifier("db", "tbl")) - .AddOption(Options::BRANCH, "dev") - .SetReadFieldNames({"name", "phone"}); + TableLoadOptions dev_load{table_path}; + dev_load.catalog = shared_catalog; + dev_load.identifier.emplace("db", "tbl", "dev"); + ScanContextBuilder dev_scan_builder; + dev_scan_builder.SetReadFieldNames({"name", "phone"}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr dev_scan_context, dev_scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr dev_scan, - TableScan::Create(std::move(dev_scan_context))); + TableScan::Create(dev_load, std::move(dev_scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr dev_plan, dev_scan->CreatePlan()); // The worker reads the same branch. - ReadContextBuilder dev_read_builder(table_path); - dev_read_builder.AddOption(Options::QUERY_AUTH_ENABLED, "true") - .AddOption(Options::BRANCH, "dev") - .SetReadFieldNames({"name", "phone"}); + TableLoadOptions dev_read_load{table_path}; + ReadContextBuilder dev_read_builder; + dev_read_builder.SetReadFieldNames({"name", "phone"}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr dev_read_context, dev_read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr dev_read, - TableRead::Create(std::move(dev_read_context))); + TableRead::Create(dev_read_load, std::move(dev_read_context), + {{Options::QUERY_AUTH_ENABLED, "true"}, + {Options::BRANCH, "dev"}})); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr dev_reader, dev_read->CreateReader(dev_plan->Splits())); @@ -636,14 +621,13 @@ This library creates no branch: one has to exist, with its schema published unde Naming the branch ^^^^^^^^^^^^^^^^^ -A branch is addressed by the identifier of ``WithCatalog``, which +A branch is addressed by ``TableLoadOptions::identifier`` or the loaded table identifier, which ``paimon::Identifier("db", "tbl", "dev")`` builds as ``tbl$branch_dev``, so that the catalog answers for the branch: the snapshot it loads is the branch's, and the commit request is addressed to that branch by that identifier rather than by -anything in its body. The ``branch`` option and, for writers, ``WithBranch`` may -name the same branch as well; naming two different branches is rejected rather +anything in its body. The dynamic ``branch`` option may name the same branch as well; naming two different branches is rejected rather than resolved, and ``tbl$branch_main``, ``branch=main`` and an empty name all -mean the main branch. Naming a branch only in the option or ``WithBranch`` while +mean the main branch. Naming a branch only in the option while the identifier names the bare table is rejected, as the catalog would then answer for the table while the files were written for the branch. @@ -669,7 +653,7 @@ A write or a commit aimed at a branch reads that schema rather than the catalog's and asks the catalog for no schema at all, so a catalog serving only the table's own current schema still takes the branch's snapshots. A scan or read also uses the fields of the schema published under the branch, but through -``WithCatalog`` it asks for the catalog's branch schema to check +a bound catalog it asks for the catalog's branch schema to check ``query-auth.enabled`` (see `Reading through the catalog`_). The new snapshot records the schema id published under the branch, which is the @@ -696,7 +680,7 @@ from a store Paimon has no client for — reach a file system through a ``REGISTER_PAIMON_FACTORY``. Build a provider with ``CredentialProviderFactory::Get``, wrap it in a ``FileSystem`` of your own that calls ``CredentialProvider::GetCredentials`` as it signs each access, and pass -that file system in through ``Catalog::Create`` or a builder's ``WithFileSystem``. +that file system in through ``Catalog::Create`` or ``TableLoadOptions::file_system``. A file system passed this way is used as-is, so the credentials are reloaded as they expire without it being rebuilt. diff --git a/docs/source/user_guide/commit.rst b/docs/source/user_guide/commit.rst index 121462cb9..c7dd46c72 100644 --- a/docs/source/user_guide/commit.rst +++ b/docs/source/user_guide/commit.rst @@ -129,7 +129,7 @@ Snapshot publication depends on the commit mode: published through an atomic rename. After publication, the committer writes the ``LATEST`` hint file to reduce list operations when fetching the latest Snapshot. -* **Catalog-mediated commit:** When ``CommitContextBuilder::WithCatalog()`` +* **Catalog-mediated commit:** When the Table is loaded through a catalog supplies a catalog that manages table versions, the committer sends the Snapshot and change statistics to that catalog for publication. This path does not rename a local Snapshot file or write a local ``LATEST`` hint. diff --git a/docs/source/user_guide/file_index.rst b/docs/source/user_guide/file_index.rst index 44955689e..a0cc67cf7 100644 --- a/docs/source/user_guide/file_index.rst +++ b/docs/source/user_guide/file_index.rst @@ -58,7 +58,7 @@ positions in that schema. For example, the following constructs PAIMON_ASSIGN_OR_RAISE( std::shared_ptr filter, paimon::PredicateBuilder::Or({text, price})); - paimon::ReadContextBuilder builder(table_path); + paimon::ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"body", "price"}) .SetPredicate(filter); @@ -82,7 +82,7 @@ predicate by projecting ``_body_INDEX_SCORE``: PAIMON_ASSIGN_OR_RAISE( std::shared_ptr text, paimon::PredicateBuilder::FullTextMatch(0, request)); - paimon::ReadContextBuilder builder(table_path); + paimon::ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"body", "_body_INDEX_SCORE"}) .SetPredicate(text); diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 5bef8f4b9..7caadce2d 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -48,33 +48,27 @@ Write and commit contexts must name the loaded table's location and, when they carry a format-table handle, its identity. Conflicting contexts are rejected before creating an operator, including for overwrite commits. -Reading and writing go through the entry points every other table uses: ``TableScan::Create``, -``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its -usual context builder. Each reads the table's schema - from under the table path, or from the one -the context carries - and dispatches to a format table when its ``type`` says so. - -That path needs a schema under the table's own location, which is what every table a file system -catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the -REST catalog, has none: nothing under the location says that it is a format table, nor that -``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written -by handing the loaded ``FormatTable`` to the context instead of a path - every context builder has -a constructor taking one - which is what Java does through ``FormatTable.newReadBuilder()`` and +Static ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` +and ``FileStoreCommit::Create`` load tables through ``TableLoadOptions``. They +read the schema from the table path or ``table_schema_json`` and dispatch by its +type. Operation Contexts contain execution settings; dynamic options are a +separate final argument. + +A format table whose schema lives only in a catalog is used through its loaded +Table handle, as Java uses ``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: .. code-block:: cpp PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, catalog->GetFormatTable(paimon::Identifier("db", "tbl"))); - paimon::ScanContextBuilder scan_builder(table); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_builder.Finish()); + paimon::ScanContextBuilder().Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); -The table already carries its schema and the file system it was loaded through, so a context built -from one refuses ``SetTableSchema()``, ``WithFileSystem()``, -``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options -given at the call still win over the ones the schema stored, as they do everywhere else. +The loaded table supplies its schema and file system. Dynamic options passed to +``New*()`` override stored options. A batch comes back in the table's column order, or the projection's when the read names one, behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for @@ -132,17 +126,15 @@ A setting the format path cannot act on is refused by name rather than quietly d * ``WriteContextBuilder::WithWriteId()``, which prefixes a postpone-bucket writer's files so one compaction reader can put them back in order; a format table has no buckets. * ``CommitContextBuilder::IgnoreEmptyCommit(false)``, ``UseRESTCatalogCommit(true)``, - ``WithTableId()``, ``WithCatalog()`` and ``AppendCommitCheckConflict(true)``. Keeping an empty - commit means writing a snapshot that adds no files, a rest-catalog commit sends that snapshot to - a catalog, a table id names the table in the request that commit sends, committing through a - catalog hands it a snapshot to take, and the conflict check reads the manifests of concurrent - commits - none of which exist here. Each is refused only when set away from its default, so an + ``TableLoadOptions::table_id`` and ``AppendCommitCheckConflict(true)``. Keeping an empty + commit means writing a snapshot that adds no files, a REST commit sends that snapshot to + a catalog, a table id names the table in that request, and the conflict check reads the + manifests of concurrent commits - none of which exist here. Each is refused only when set away from its default, so an ordinary commit is unaffected. * the ``branch`` option, whatever names it: a format table keeps no metadata to branch, and its data is the files under its one location, which is where a read or a write would go whichever branch was asked for. It is refused wherever it comes from - the schema, a catalog that parsed - ``tbl$branch_dev``, or the options of a single call - and so is ``WithBranch()`` on a context - built from a format table. A branch quietly ignored would be the wrong answer everywhere and a + ``tbl$branch_dev``, or the dynamic options of a single call. A branch quietly ignored would be the wrong answer everywhere and a destructive one for an overwrite, which would clear the data it was told not to touch. The value naming the main branch is the default and is left alone. * A scan predicate or bucket filter, as above, and more than one partition filter: a scan descends diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 5fcc3b1e2..1cb7e8c2f 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -223,10 +223,15 @@ to both the index scan and the data scan: reader->VisitFullTextSearch(std::make_shared( "f0", R"({"match":{"query":"paimon lake"}})", /*limit=*/10))); - paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(paimon::Options::SCAN_MODE, "from-snapshot") - .AddOption(paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) - .SetGlobalIndexResult(result); + paimon::TableLoadOptions load_options{table_path}; + paimon::ScanContextBuilder scan_builder; + const std::map scan_options = { + {paimon::Options::SCAN_MODE, "from-snapshot"}, + {paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)}}; + scan_builder.SetGlobalIndexResult(result); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + paimon::TableScan::Create(load_options, std::move(context), scan_options)); **Configuration:** @@ -313,20 +318,21 @@ them. A search whose ``full-text-index.search-mode``, or else ``global-index.sea search_builder.ExecuteLocal()); // Read the rows found, with their scores. - paimon::ScanContextBuilder scan_builder(table_path); + paimon::TableLoadOptions load_options{table_path}; + paimon::ScanContextBuilder scan_builder; scan_builder.SetGlobalIndexResult(result); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + paimon::TableScan::Create(load_options, std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_plan, scan->CreatePlan()); - paimon::ReadContextBuilder read_builder(table_path); + paimon::ReadContextBuilder read_builder; read_builder.SetReadFieldNames({"id", "content", "_INDEX_SCORE"}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - paimon::TableRead::Create(std::move(read_context))); + paimon::TableRead::Create(load_options, std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(data_plan->Splits())); @@ -416,12 +422,17 @@ outlive the snapshot. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, search_read->Read(*plan)); - paimon::ScanContextBuilder scan_builder(table_path); + paimon::TableLoadOptions load_options{table_path}; + paimon::ScanContextBuilder scan_builder; scan_builder.SetGlobalIndexResult(result); + std::map scan_options; if (plan->SnapshotId()) { - scan_builder.AddOption(paimon::Options::SCAN_MODE, "from-snapshot") - .AddOption(paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(*plan->SnapshotId())); + scan_options[paimon::Options::SCAN_MODE] = "from-snapshot"; + scan_options[paimon::Options::SCAN_SNAPSHOT_ID] = std::to_string(*plan->SnapshotId()); } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + paimon::TableScan::Create(load_options, std::move(context), scan_options)); ``ExecuteLocal()`` plans and searches in one process. To plan in one process and search in others, plan with ``NewFullTextScan()``, ship the splits with ``FullTextSearchSplit::Serialize()`` and diff --git a/docs/source/user_guide/manifest_cache.rst b/docs/source/user_guide/manifest_cache.rst index 6b45fe00c..23c7703c4 100644 --- a/docs/source/user_guide/manifest_cache.rst +++ b/docs/source/user_guide/manifest_cache.rst @@ -86,7 +86,7 @@ Example: std::make_shared(), std::make_shared()); - paimon::ScanContextBuilder scan_builder(table_path); + paimon::ScanContextBuilder scan_builder; scan_builder.WithCache(cache); Passing ``nullptr`` or omitting ``ScanContextBuilder::WithCache()`` leaves diff --git a/docs/source/user_guide/manifest_entry_cache.rst b/docs/source/user_guide/manifest_entry_cache.rst index 4d9fed056..ab55b230d 100644 --- a/docs/source/user_guide/manifest_entry_cache.rst +++ b/docs/source/user_guide/manifest_entry_cache.rst @@ -78,13 +78,15 @@ under .. code-block:: cpp auto cache = std::make_shared(128 * 1024 * 1024); - ScanContextBuilder context_builder(table_path); + TableLoadOptions load_options{table_path}; + ScanContextBuilder context_builder; PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scan_context, - context_builder - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .Finish()); + context_builder.WithCache(cache).Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + TableScan::Create(load_options, std::move(scan_context), + {{Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"}})); .. note:: diff --git a/docs/source/user_guide/parquet_metadata_cache.rst b/docs/source/user_guide/parquet_metadata_cache.rst index 514f1b64c..80c7da0fd 100644 --- a/docs/source/user_guide/parquet_metadata_cache.rst +++ b/docs/source/user_guide/parquet_metadata_cache.rst @@ -91,10 +91,10 @@ Example: std::make_shared(), std::make_shared()); - paimon::ScanContextBuilder scan_builder(table_path); + paimon::ScanContextBuilder scan_builder; scan_builder.WithCache(cache); - paimon::ReadContextBuilder read_builder(table_path); + paimon::ReadContextBuilder read_builder; read_builder.WithCache(cache); Passing ``nullptr`` or omitting ``WithCache()`` leaves Parquet metadata caching diff --git a/docs/source/user_guide/read.rst b/docs/source/user_guide/read.rst index d55a141c1..76543e074 100644 --- a/docs/source/user_guide/read.rst +++ b/docs/source/user_guide/read.rst @@ -49,10 +49,11 @@ In a function returning ``Status`` or ``Result``, use the ordinary scan API: .. code-block:: cpp - ScanContextBuilder builder(table_path); + TableLoadOptions load_options{table_path}; + ScanContextBuilder builder; builder.WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, TableScan::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, TableScan::Create(load_options, std::move(context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); * The pool controls only allocations routed through it, not all planning memory diff --git a/docs/source/user_guide/realtime.rst b/docs/source/user_guide/realtime.rst index b32454148..74ee104b0 100644 --- a/docs/source/user_guide/realtime.rst +++ b/docs/source/user_guide/realtime.rst @@ -291,34 +291,34 @@ and an input ``batch`` built as described above. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime, RealtimeContext::Create()); - WriteContextBuilder write_builder(table_path, commit_user); - write_builder.SetOptions(options) - .WithStreamingMode(true) + TableLoadOptions load_options{table_path}; + WriteContextBuilder write_builder(commit_user); + write_builder.WithStreamingMode(true) .WithRealtimeContext(realtime); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(load_options, std::move(write_context), options)); PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch))); To query before preparing or committing: .. code-block:: cpp - ScanContextBuilder scan_builder(table_path); - scan_builder.SetOptions(options).WithRealtimeContext(realtime); + ScanContextBuilder scan_builder; + scan_builder.WithRealtimeContext(realtime); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(load_options, std::move(scan_context), options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); - ReadContextBuilder read_builder(table_path); - read_builder.SetOptions(options).WithRealtimeContext(realtime); + ReadContextBuilder read_builder; + read_builder.WithRealtimeContext(realtime); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(load_options, std::move(read_context), options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); // Consume NextBatch() or NextBatchWithBitmap(), then Close() the reader. @@ -336,11 +336,11 @@ Checkpoint with progress-aware APIs: PAIMON_ASSIGN_OR_RAISE(std::vector progress, writer->PrepareCommitWithProgress(checkpoint_id)); - CommitContextBuilder commit_builder(table_path, commit_user); + CommitContextBuilder commit_builder(commit_user); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_builder.SetOptions(options).Finish()); + commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(load_options, std::move(commit_context), options)); PAIMON_ASSIGN_OR_RAISE( int64_t snapshot_id, commit->CommitWithProgress(progress, checkpoint_id, /*watermark=*/std::nullopt)); @@ -373,7 +373,8 @@ deduplicate ``deleted_offsets`` bitmap. Preserve the complete progress when coordinating checkpoints. Snapshot conflicts are retried internally against the latest snapshot; commit identifiers must follow the streaming commit protocol. With a version-managed catalog, configure the writer and committer -with the same ``WithCatalog()`` catalog and identifier. +from the same catalog-loaded Table, or use the same catalog and identifier in +``TableLoadOptions``. Read Views and Consistency -------------------------- diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst index 0acce59a4..9ac324226 100644 --- a/docs/source/user_guide/write.rst +++ b/docs/source/user_guide/write.rst @@ -256,7 +256,7 @@ produce a correct ``Snapshot``, which commonly includes (but is not limited to): file-system commits for both table types on non-object-store paths. Object-store paths need a commit that does not rename: either a catalog that manages the versions of its tables, such as the REST catalog, passed to - ``CommitContextBuilder::WithCatalog()``, or ``UseRESTCatalogCommit(true)``, + a table loaded through a version-managed catalog, or ``UseRESTCatalogCommit(true)``, which builds the commit request for the caller to send instead of publishing the snapshot itself. The two are mutually exclusive. A catalog without version management still commits by renaming, so passing one is rejected on @@ -301,7 +301,7 @@ Operational Flow passes the objects to ``FileStoreCommit`` for either an Append or PK table. For a table of a catalog which owns its versions, such as a REST catalog, it passes that catalog and the table identifier to - ``WriteContextBuilder::WithCatalog`` and ``CommitContextBuilder::WithCatalog``. + a shared catalog-loaded Table or matching ``TableLoadOptions``. The writer loads the latest snapshot from the catalog, and the current schema from it as well when it writes to the main branch; a write aimed at another branch reads the schema published under ``branch/branch-`` instead. On diff --git a/examples/clean_demo.cpp b/examples/clean_demo.cpp index b2888a86a..c5e4e1ecb 100644 --- a/examples/clean_demo.cpp +++ b/examples/clean_demo.cpp @@ -26,6 +26,7 @@ #include "arrow/ipc/api.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/orphan_files_cleaner.h" namespace fs = std::filesystem; @@ -47,27 +48,36 @@ Status CleanOrphanFiles(const std::string& table_path, int64_t older_than_ms) { Status DropPartition(const std::string& table_path, const std::vector>& partitions) { - CommitContextBuilder commit_context_builder(table_path, /*commit_user=*/"commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(/*commit_user=*/"commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr committer, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); PAIMON_RETURN_NOT_OK(committer->DropPartition(partitions, /*commit_identifier=*/10)); return Status::OK(); } Status ExpireSnapshot(const std::string& table_path) { - CommitContextBuilder commit_context_builder(table_path, /*commit_user=*/"commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(/*commit_user=*/"commit_user_1"); std::map commit_options = { {Options::SNAPSHOT_NUM_RETAINED_MAX, "2"}, {Options::SNAPSHOT_NUM_RETAINED_MIN, "1"}, {Options::SNAPSHOT_TIME_RETAINED, "1ms"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}}; + commit_context_builder_options = commit_options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(commit_options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr committer, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); PAIMON_RETURN_NOT_OK(committer->Expire()); return Status::OK(); diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index df46fcc54..51ddc027d 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -27,6 +27,7 @@ #include "paimon/api.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/table.h" +#include "paimon/catalog/table_load_options.h" arrow::Result> PrepareData(const arrow::FieldVector& fields) { arrow::StringBuilder f0_builder; @@ -82,8 +83,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - // The table every operator below is created from. Each call hands the table's shared owner - // to the operator it creates. + // Operators use this loaded table's schema and options. Use Table::Copy to override options. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, catalog->GetTable(paimon::Identifier(db_name, table_name))); @@ -91,9 +91,9 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, std::string commit_user = "some_commit_user"; // write - paimon::WriteContextBuilder context_builder(table_path, commit_user); + paimon::WriteContextBuilder context_builder(commit_user); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, table->NewWrite(std::move(write_context))); @@ -115,25 +115,25 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, writer->PrepareCommit()); // commit - paimon::CommitContextBuilder commit_context_builder(table_path, commit_user); + paimon::CommitContextBuilder commit_context_builder(commit_user); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); + commit_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, table->NewCommit(std::move(commit_context))); PAIMON_RETURN_NOT_OK(committer->Commit(commit_message)); // scan - paimon::ScanContextBuilder scan_context_builder(table_path); + paimon::ScanContextBuilder scan_context_builder; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_context_builder.SetOptions(options).Finish()); + scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scanner, table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scanner->CreatePlan()); // read - paimon::ReadContextBuilder read_context_builder(table_path); + paimon::ReadContextBuilder read_context_builder; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, - read_context_builder.SetOptions(options).Finish()); + read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index ddb1c6ad4..112c1010e 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -71,8 +71,7 @@ class PAIMON_EXPORT Table { return uuid_.empty() ? FullName() : uuid_; } - /// Returns the metastore UUID, or null if unavailable, for - /// `CommitContextBuilder::WithTableId()`. + /// Returns the metastore UUID, or null if unavailable. std::optional CatalogUuid() const { return uuid_.empty() ? std::nullopt : std::optional(uuid_); } @@ -83,6 +82,12 @@ class PAIMON_EXPORT Table { return schema_; } + /// Returns an independent table with dynamic options applied to its bound configuration. + /// The original table is unchanged. Type, location, branch, catalog identity and storage + /// bindings are retained; load another table to select a different branch. + virtual Result> Copy( + const std::map& dynamic_options) const = 0; + /// Creates a scan using this table's schema and options. virtual Result> NewScan( std::unique_ptr context) const = 0; diff --git a/include/paimon/catalog/table_load_options.h b/include/paimon/catalog/table_load_options.h new file mode 100644 index 000000000..3203f41c9 --- /dev/null +++ b/include/paimon/catalog/table_load_options.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/visibility.h" + +namespace paimon { +class Catalog; +class FileSystem; + +/// Parameters used only while loading a table for a static operator factory. +/// A catalog requires an identifier; a path is required for file-system loading. +struct PAIMON_EXPORT TableLoadOptions { + /// Optional for catalog loads; otherwise the native table or system-table path. + std::string path; + /// Explicit storage binding, taking precedence over the catalog's per-table file system. + std::shared_ptr file_system; + /// URI scheme routing; cannot be specified together with file_system. + std::map fs_scheme_to_identifier_map; + /// Serialized Paimon table schema, independent of the reader's Arrow projection. + /// Used for the main branch, including a fallback to main from another branch. + std::optional table_schema_json; + std::shared_ptr catalog; + /// Requires catalog. A named branch must agree with the dynamic branch option. + std::optional identifier; + /// Expected catalog UUID, or the UUID for an external REST commit request. + std::optional table_id; +}; +} // namespace paimon diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index ffd40ac07..c0f68049f 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -19,20 +19,15 @@ #pragma once -#include #include -#include #include -#include "paimon/catalog/identifier.h" #include "paimon/result.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" namespace paimon { -class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `CommitContext` is some configuration for commit operations. @@ -42,31 +37,12 @@ class MemoryPool; /// @see CommitContextBuilder class PAIMON_EXPORT CommitContext { public: - /// Retained for source compatibility. Prefer `CommitContextBuilder` for input validation. - CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - bool append_commit_check_conflict, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); - - CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - const std::shared_ptr& catalog, - const std::optional& identifier, - const std::optional& table_id, bool append_commit_check_conflict, + CommitContext(const std::string& commit_user, bool ignore_empty_commit, + bool use_rest_catalog_commit, bool append_commit_check_conflict, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::shared_ptr& executor); ~CommitContext(); - const std::string& GetRootPath() const { - return root_path_; - } - const std::string& GetCommitUser() const { return commit_user_; } @@ -79,21 +55,6 @@ class PAIMON_EXPORT CommitContext { return use_rest_catalog_commit_; } - /// Returns the configured catalog, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - - /// Returns the catalog table UUID sent as `tableId` in commit requests. - const std::optional& GetTableId() const { - return table_id_; - } - bool AppendCommitCheckConflict() const { return append_commit_check_conflict_; } @@ -106,73 +67,22 @@ class PAIMON_EXPORT CommitContext { return executor_; } - std::shared_ptr GetSpecificFileSystem() const { - return specific_file_system_; - } - - const std::map& GetOptions() const { - return options_; - } - - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - private: - std::string root_path_; std::string commit_user_; bool ignore_empty_commit_; bool use_rest_catalog_commit_; - std::shared_ptr catalog_; - std::optional identifier_; - std::optional table_id_; bool append_commit_check_conflict_; std::shared_ptr memory_pool_; std::shared_ptr executor_; - std::shared_ptr specific_file_system_; - std::map options_; - std::shared_ptr format_table_; }; /// `CommitContextBuilder` used to build a `CommitContext`, has input validation. class PAIMON_EXPORT CommitContextBuilder { public: - /// Constructs a `CommitContextBuilder` with required parameters. - /// @param root_path The root path of the Paimon table. - /// @param commit_user The user identifier for the commit operation. - CommitContextBuilder(const std::string& root_path, const std::string& commit_user); - - /// Constructs a `CommitContextBuilder` for a format table that is already loaded: the only way - /// to commit to one whose schema lives in a metastore rather than under its location, such as - /// a table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()` is refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to commit to, as `Catalog::GetFormatTable()` hands it back. - explicit CommitContextBuilder(const std::shared_ptr& table); + explicit CommitContextBuilder(const std::string& commit_user); ~CommitContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Sets whether to ignore empty commits (default is true). /// When set to true, commits that don't contain any actual data changes will be ignored. /// @param ignore_empty_commit True to ignore empty commits, false otherwise. @@ -180,39 +90,12 @@ class PAIMON_EXPORT CommitContextBuilder { CommitContextBuilder& IgnoreEmptyCommit(bool ignore_empty_commit); /// Builds a REST commit request for the caller to send (default is false). - /// Use `WithCatalog()` for catalog-managed commits; the two modes are mutually exclusive. + /// A table loaded through a catalog commits through that catalog; the two modes are exclusive. /// @note Temporary interface, will be removed in the future. /// @param use_rest_catalog_commit True to build the request without sending it. /// @return Reference to this builder for method chaining. CommitContextBuilder& UseRESTCatalogCommit(bool use_rest_catalog_commit); - /// Loads the current schema from `catalog`. Catalogs with version management also load and - /// commit snapshots, rebasing on conflicts. Other catalogs use file-system snapshot commits. - /// Mutually exclusive with `UseRESTCatalogCommit()`. - /// - /// Data and manifests remain on the file system. Recovery, rollback and row-id checks require - /// historical snapshots there; row-id checks also require the inspected data files' schemas. - /// `Expire()` manages file-system snapshots after confirming that the catalog's current - /// snapshot and retained history are published there. Configure writers with the same catalog - /// via `WriteContextBuilder::WithCatalog()`. - /// A branch is addressed by the identifier, as `tbl$branch_dev`, so that the catalog answers - /// for that branch. Its schema is read from the branch directory rather than from the catalog, - /// just as a read of that branch reads it. Configure the writer with the same identifier. - /// @param catalog Non-null catalog, kept alive by this context. - /// @param identifier Table to commit to, naming a branch of it as `tbl$branch_dev`. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier); - - /// Sets the catalog table UUID sent as `tableId` to detect table recreation (default is unset). - /// - /// Capture `Table::CatalogUuid()` before preparing changes. `Table::Uuid()` may fall back to - /// the table name. An unset ID is sent as null and validated by the server. - /// Requires `WithCatalog()` or `UseRESTCatalogCommit()`. - /// @param table_id Catalog UUID of the table to commit to. - /// @return Reference to this builder for method chaining. - CommitContextBuilder& WithTableId(const std::string& table_id); - /// Sets whether append commits should perform conflict checking (default is false). /// @param append_commit_check_conflict True to enable append conflict checks. /// @return Reference to this builder for method chaining. @@ -228,15 +111,6 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. CommitContextBuilder& WithExecutor(const std::shared_ptr& executor); - /// Sets a custom file system instance to be used for all file operations in this commit - /// context. - /// This bypasses the global file system registry and uses the provided implementation directly. - /// - /// @param file_system The file system to use. - /// @return Reference to this builder for method chaining. - /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) - CommitContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - /// Build and return a `CommitContext` instance with input validation. /// @return Result containing the constructed `CommitContext` or an error status. Result> Finish(); diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index fcb094e05..0a1b505b4 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -26,6 +26,7 @@ #include #include +#include "paimon/catalog/table_load_options.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" @@ -46,16 +47,20 @@ class CommitMessage; /// dropping partitions, and retrieving commit metrics. /// /// @note Direct file-system commits support append-only and primary-key tables on non-object-store -/// paths. `CommitContextBuilder::WithCatalog()` also supports object stores when the catalog +/// paths. `TableLoadOptions::catalog` also supports object stores when the catalog /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: /// Create an instance of `FileStoreCommit`. /// + /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `CommitContext` used for commit operations. /// /// @return A Result containing a unique pointer to the `FileStoreCommit` instance. - static Result> Create(std::unique_ptr context); + static Result> Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options = {}); virtual ~FileStoreCommit() = default; @@ -93,10 +98,10 @@ class PAIMON_EXPORT FileStoreCommit { /// An error is terminal for the writer state which produced `realtime_commits`. The caller must /// discard its `RealtimeContext` and `FileStoreWrite`, load the current latest snapshot's /// durable offsets, recreate both objects, and replay input from those exclusive offsets. - /// Conflicts from `CommitContextBuilder::WithCatalog()` are retried internally. With + /// Conflicts from `TableLoadOptions::catalog` are retried internally. With /// `UseRESTCatalogCommit()`, the caller sends the request and handles server conflicts. /// Concurrent rollback or partition deletion must be fenced by the upstream coordinator. - /// Configure catalog writers with `WriteContextBuilder::WithCatalog()` too. + /// Load writers through the same catalog using TableLoadOptions or Catalog::GetTable(). /// /// In deduplicate mode each commit attempt resolves logical deleted offsets against the /// latest snapshot and merges them with its deletion vectors. Keep the complete progress diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index ae01dd080..4205df0e7 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -25,6 +25,7 @@ #include #include +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_message.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -48,10 +49,14 @@ class PAIMON_EXPORT FileStoreWrite { public: /// Create an instance of `FileStoreWrite`. /// + /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `WriteContext` used for write operations. /// /// @return A Result containing a unique pointer to the `FileStoreWrite` instance. - static Result> Create(std::unique_ptr context); + static Result> Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options = {}); virtual ~FileStoreWrite() = default; @@ -142,7 +147,7 @@ class PAIMON_EXPORT FileStoreWrite { /// Snapshot installation and queue cleanup are serialized with `Seal()` and /// `PrepareCommitWithProgress()`. /// - /// With `WriteContextBuilder::WithCatalog()`, the latest snapshot can be loaded from the + /// When the table was loaded through a catalog, the latest snapshot can be loaded from the /// catalog. Older requested snapshots must be available on the file system. /// /// If the snapshot overwrites table contents or moves committed progress backwards, such as diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 95aafb8f0..7170b3046 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -19,15 +19,12 @@ #pragma once #include -#include #include -#include #include #include #include "arrow/c/abi.h" #include "paimon/cache/cache.h" -#include "paimon/catalog/identifier.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -35,13 +32,10 @@ #include "paimon/visibility.h" namespace paimon { -class Catalog; class Executor; struct FullTextSearch; -class FormatTable; class MemoryPool; class Predicate; -class FileSystem; class RealtimeContext; struct VectorSearch; @@ -52,8 +46,7 @@ struct VectorSearch; /// @see ReadContextBuilder class PAIMON_EXPORT ReadContext { public: - ReadContext(const std::string& path, const std::string& branch, - const std::vector& read_field_names, + ReadContext(const std::vector& read_field_names, const std::vector& read_field_ids, const std::shared_ptr& predicate, const std::shared_ptr& vector_search, @@ -61,36 +54,14 @@ class PAIMON_EXPORT ReadContext { bool enable_precise_file_index_evaluate, bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, - uint32_t row_to_batch_thread_number, const std::optional& table_schema, - const std::shared_ptr& memory_pool, + uint32_t row_to_batch_thread_number, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, - const std::optional& identifier); + bool read_ahead_cache_enabled, const CacheConfig& cache_config, + const std::shared_ptr& cache, WarmupLevel warmup_level); ~ReadContext(); - const std::string& GetPath() const { - return path_; - } - - const std::string& GetBranch() const { - return branch_; - } - - const std::map& GetFileSystemSchemeToIdentifierMap() const { - return fs_scheme_to_identifier_map_; - } - - const std::map& GetOptions() const { - return options_; - } - const std::vector& GetReadFieldNames() const { return read_field_names_; } @@ -141,18 +112,13 @@ class PAIMON_EXPORT ReadContext { uint32_t GetRowToBatchThreadNumber() const { return row_to_batch_thread_number_; } - const std::optional& GetSpecificTableSchema() const { - return table_schema_; - } + std::shared_ptr GetMemoryPool() const { return memory_pool_; } std::shared_ptr GetExecutor() const { return executor_; } - std::shared_ptr GetSpecificFileSystem() const { - return specific_file_system_; - } /// Returns the context used to resolve process-local real-time split tickets. std::shared_ptr GetRealtimeContext() const { @@ -171,26 +137,10 @@ class PAIMON_EXPORT ReadContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - WarmupLevel GetWarmupLevel() const { return warmup_level_; } - /// Returns the catalog supplying table metadata, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. bool HasReadSchema() const { return read_schema_ != nullptr && read_schema_->release != nullptr; @@ -208,8 +158,6 @@ class PAIMON_EXPORT ReadContext { void SetReadSchema(std::unique_ptr schema); private: - std::string path_; - std::string branch_; std::vector read_field_names_; std::vector read_field_ids_; std::shared_ptr predicate_; @@ -223,20 +171,13 @@ class PAIMON_EXPORT ReadContext { uint32_t prefetch_max_parallel_num_; bool enable_multi_thread_row_to_batch_; uint32_t row_to_batch_thread_number_; - std::optional table_schema_; std::shared_ptr memory_pool_; std::shared_ptr executor_; - std::shared_ptr specific_file_system_; - std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; - std::map options_; bool read_ahead_cache_enabled_; CacheConfig cache_config_; std::shared_ptr cache_; - std::shared_ptr format_table_; WarmupLevel warmup_level_; - std::shared_ptr catalog_; - std::optional identifier_; // Owns schema resources and releases ArrowSchema::release in destructor. std::unique_ptr read_schema_; }; @@ -244,18 +185,7 @@ class PAIMON_EXPORT ReadContext { /// `ReadContextBuilder` used to build a `ReadContext`, has input validation. class PAIMON_EXPORT ReadContextBuilder { public: - /// Constructs a `ReadContextBuilder` with required parameters. - /// @param path The root path of the table. - explicit ReadContextBuilder(const std::string& path); - - /// Constructs a `ReadContextBuilder` for a format table that is already loaded: the only way - /// to read one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a - /// branch are refused here rather than ignored. - /// - /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back. - explicit ReadContextBuilder(const std::shared_ptr& table); + ReadContextBuilder(); ~ReadContextBuilder(); @@ -324,7 +254,7 @@ class PAIMON_EXPORT ReadContextBuilder { /// return paimon::Status::Invalid(export_status.message()); /// } /// - /// ReadContextBuilder builder("/path/to/table"); + /// ReadContextBuilder builder; /// builder.SetReadSchema(std::move(c_schema)); /// @endcode /// @@ -335,23 +265,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - ReadContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - ReadContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a predicate for filtering data during reading. /// /// The predicate is used for both partition pruning and data filtering. @@ -516,77 +429,6 @@ class PAIMON_EXPORT ReadContextBuilder { ReadContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Set the table schema as a string to avoid schema loading I/O operations. - /// - /// This optimization allows the reader to use a pre-loaded schema instead of - /// reading it from the table metadata, which can improve performance especially - /// in scenarios with many small read operations. - /// - /// @param table_schema String representation of the table schema. - /// @return Reference to this builder for method chaining. - /// @note The user must ensure that the schema string is valid and matches the table. - /// @note If not set, the schema will be loaded from the table path. - ReadContextBuilder& SetTableSchema(const std::string& table_schema); - - /// Set the specific branch to read from in a versioned table. - /// - /// Paimon supports branching for data versioning and time travel queries. - /// This method allows reading from a specific branch instead of the main branch. - /// - /// The `branch` option names the branch too, as it does for a scan of one branch; naming two - /// different branches is refused rather than silently resolved. - /// - /// @param branch Name of the branch to read from. - /// @return Reference to this builder for method chaining. - /// @note Default branch is "main" if not specified. An empty name is the main branch. - ReadContextBuilder& WithBranch(const std::string& branch); - - /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system - /// identifiers. This allows selecting different pre-registered file system implementations - /// based on the URI scheme at runtime. - /// - /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding - /// file system identifier. - /// @return Reference to this builder for method chaining. - /// @note - /// - This method is intended for environments where multiple file systems are pre-registered. - /// - The specified identifiers must correspond to file systems that have been registered at - /// compile time or initialization. - /// - Cannot be used together with `WithFileSystem()`. - /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). - /// Example: - /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); - /// - ReadContextBuilder& WithFileSystemSchemeToIdentifierMap( - const std::map& fs_scheme_to_identifier_map); - - /// Sets a custom file system instance to be used for all file operations in this read context. - /// This bypasses the global file system registry and uses the provided implementation directly. - /// - /// @param file_system The file system to use. - /// @return Reference to this builder for method chaining. - /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) - ReadContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - - /// Reads a native table through its catalog, which supplies its file system: - /// `catalog->GetTableFileSystem(identifier)`, with any per-table temporary credentials; an - /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked for one. - /// `TableRead::Create()` reads `query-auth.enabled` from the schema the catalog holds for the - /// branch read. A branch named by a system table path is resolved before requesting the table's - /// file system and must agree with the identifier, `branch` option and `WithBranch()`. - /// The fields the read reads with still come from `SetTableSchema()` on the main - /// branch or from the latest schema under the table path. When the catalog does not enable it, - /// the read path must be `catalog->GetTableLocation()` of the branch read. - /// A `sys` identifier requires a global system table path. A branch the catalog - /// would read as another object, such as `MAIN` or one holding a `$`, is rejected before the - /// catalog is asked for anything. - /// @param catalog Non-null catalog, read when `Finish()` builds the context and when - /// `TableRead::Create()` runs. - /// @param identifier The native table to read. - /// @return Reference to this builder for method chaining. - ReadContextBuilder& WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier); - /// Inject a cache for read operations. Passing nullptr disables cache. /// @return Reference to this builder for method chaining. ReadContextBuilder& WithCache(const std::shared_ptr& cache); diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 93f3ba2db..730ff8968 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -26,7 +26,6 @@ #include #include "paimon/cache/cache.h" -#include "paimon/catalog/identifier.h" #include "paimon/global_index/global_index_result.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" @@ -35,9 +34,7 @@ namespace paimon { class ScanContextBuilder; class ScanFilter; -class Catalog; class Executor; -class FormatTable; class MemoryPool; class Predicate; @@ -48,42 +45,16 @@ class Predicate; /// @see ScanContextBuilder class PAIMON_EXPORT ScanContext { public: - ScanContext(const std::string& path, bool is_streaming_mode, std::optional limit, + ScanContext(bool is_streaming_mode, std::optional limit, const std::shared_ptr& scan_filter, const std::shared_ptr& global_index_result, const std::shared_ptr& realtime_context, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& format_table, - const std::shared_ptr& catalog, - const std::optional& identifier); - - /// Also takes the columns the query reads, see `ScanContextBuilder::SetReadFieldNames()`. - ScanContext(const std::string& path, bool is_streaming_mode, std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& format_table, - const std::shared_ptr& catalog, - const std::optional& identifier, - const std::optional>& read_field_names); + const std::shared_ptr& executor, const std::shared_ptr& cache, + const std::optional>& read_field_names = std::nullopt); ~ScanContext(); - const std::string& GetPath() const { - return path_; - } - bool IsStreamingMode() const { return is_streaming_mode_; } @@ -95,9 +66,6 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr GetScanFilters() const { return scan_filters_; } - const std::map& GetOptions() const { - return options_; - } std::shared_ptr GetMemoryPool() const { return memory_pool_; @@ -115,40 +83,15 @@ class PAIMON_EXPORT ScanContext { return realtime_context_; } - std::shared_ptr GetSpecificFileSystem() const { - return specific_file_system_; - } - - const std::optional& GetSpecificTableSchema() const { - return table_schema_; - } - std::shared_ptr GetCache() const { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - - /// Returns the catalog supplying table metadata, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - const std::optional>& GetReadFieldNames() const { return read_field_names_; } private: - std::string path_; bool is_streaming_mode_; std::optional limit_; std::shared_ptr scan_filters_; @@ -156,13 +99,7 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr realtime_context_; std::shared_ptr memory_pool_; std::shared_ptr executor_; - std::shared_ptr specific_file_system_; - std::optional table_schema_; - std::map options_; std::shared_ptr cache_; - std::shared_ptr format_table_; - std::shared_ptr catalog_; - std::optional identifier_; std::optional> read_field_names_; }; @@ -195,17 +132,7 @@ class PAIMON_EXPORT ScanFilter { /// `ScanContextBuilder` used to build a `ScanContext`, has input validation. class PAIMON_EXPORT ScanContextBuilder { public: - /// Constructs a `ScanContextBuilder` with required parameters. - /// @param path The root path of the table. - explicit ScanContextBuilder(const std::string& path); - - /// Constructs a `ScanContextBuilder` for a format table that is already loaded: the only way - /// to scan one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. - /// - /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back. - explicit ScanContextBuilder(const std::shared_ptr& table); + ScanContextBuilder(); ~ScanContextBuilder(); /// If limit is not set, it defaults to unlimited. @@ -234,16 +161,6 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// The options added or set in `ScanContextBuilder` have high priority and will be merged with - /// the options in table schema. - ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - ScanContextBuilder& SetOptions(const std::map& options); - /// Set whether the scan is in streaming mode. /// @note if not set, is_streaming_mode = false ScanContextBuilder& WithStreamingMode(bool is_streaming_mode); @@ -260,46 +177,6 @@ class PAIMON_EXPORT ScanContextBuilder { /// @return Reference to this builder for method chaining. /// @note if not set, executor is default executor ScanContextBuilder& WithExecutor(const std::shared_ptr& executor); - /// Sets a custom file system instance to be used for all file operations in this scan context. - /// This bypasses the global file system registry and uses the provided implementation directly. - /// @param file_system The file system to use. - /// @return Reference to this builder for method chaining. - /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) - ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - - /// Plans a native table through its catalog, which supplies its file system and, for a table - /// with `query-auth.enabled`, its rules. The file system is - /// `catalog->GetTableFileSystem(identifier)`, with any per-table temporary credentials; an - /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked for one. - /// `TableScan::Create()` reads `query-auth.enabled` from the schema the catalog holds for the - /// branch scanned; the fields the scan plans with still come from `SetTableSchema()` on the - /// main branch or from the latest schema under the table path. The scan path must be - /// `catalog->GetTableLocation()` of the branch scanned, so the catalog's setting is taken only - /// for the table it keeps there. A `sys` identifier requires a global system table path. - /// A query-auth scan checks the location again on every plan, along with whether its selected - /// and rule columns keep their field ids in the catalog's current schema. - /// A branch the `branch` option or a system table path names is carried into the identifier, - /// as `tbl$branch_dev`, before requesting the table's file system; an - /// identifier naming another branch, or a branch the catalog would read as another object, such - /// as `MAIN` or one holding a `$`, is rejected before the catalog is asked for anything. - /// @param catalog Non-null catalog, read when `Finish()` builds the context and when - /// `TableScan::Create()` runs. - /// @param identifier The native table to scan. - /// @return Reference to this builder for method chaining. - ScanContextBuilder& WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier); - - /// Set the table schema as a string to avoid schema loading I/O operations. - /// - /// This optimization allows the scanner to use a pre-loaded schema instead of - /// reading it from the table metadata, which can improve performance especially - /// in scenarios with many small scan operations. - /// - /// @param table_schema String representation of the table schema. - /// @return Reference to this builder for method chaining. - /// @note The user must ensure that the schema string is valid and matches the table. - /// @note If not set, the schema will be loaded from the table path. - ScanContextBuilder& SetTableSchema(const std::string& table_schema); /// Inject a cache for scan operations. Passing nullptr disables cache. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index 4d9382566..da6be3971 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -50,8 +50,8 @@ class FileSystem; /// /// Reading and writing go through the entry points every other table uses: `TableScan`, /// `TableRead`, `FileStoreWrite` and `FileStoreCommit`. Each recognises a format table from the -/// schema under the table path, or takes one already loaded through the `FormatTable` constructor -/// every context builder has, which is how a table whose schema lives in a metastore is reached. +/// schema under the table path, or uses NewScan/NewRead/NewWrite/NewCommit on a table already +/// loaded through its catalog. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. class PAIMON_EXPORT FormatTable : public Table { @@ -103,9 +103,8 @@ class PAIMON_EXPORT FormatTable : public Table { bool location_carries_paimon_metadata, const std::map& dynamic_options); - /// Copies `table` with `dynamic_options` on top of the options it already carries, which is - /// the precedence every context builder promises for a table it was handed rather than loaded - /// itself. `table` comes back as it is when there is nothing to add. + /// Copies `table` with `dynamic_options` overriding the options it already carries. + /// Returns `table` itself when there is nothing to add. /// /// `type` is not overridable: it is structural and is read from the schema alone. /// @@ -118,6 +117,9 @@ class PAIMON_EXPORT FormatTable : public Table { ~FormatTable() override; + Result> Copy( + const std::map& dynamic_options) const override; + Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; Result> NewWrite( @@ -194,10 +196,6 @@ class PAIMON_EXPORT FormatTable : public Table { } private: - Status CheckWriteContext(const std::string& path, - const std::shared_ptr& context_table) const; - Result> ForContext( - const std::map& options) const; FormatTable(const std::shared_ptr& file_system, const std::string& location, const Identifier& identifier, const std::shared_ptr& schema, const std::map& options, Format format, diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index c77b323da..46485459a 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -22,6 +22,7 @@ #include #include +#include "paimon/catalog/table_load_options.h" #include "paimon/executor.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" @@ -44,9 +45,13 @@ class PAIMON_EXPORT TableRead { /// Create an instance of `TableRead`. /// + /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. - static Result> Create(std::unique_ptr context); + static Result> Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options = {}); /// Creates a `BatchReader` instance for reading data. /// diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 8bae74c41..e02a059f9 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -24,6 +24,7 @@ #include #include +#include "paimon/catalog/table_load_options.h" #include "paimon/result.h" #include "paimon/table/source/plan.h" #include "paimon/table/source/scan_metrics.h" @@ -38,9 +39,13 @@ class PAIMON_EXPORT TableScan { public: /// Create an instance of `TableScan`. /// + /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. - static Result> Create(std::unique_ptr context); + static Result> Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options = {}); virtual ~TableScan() = default; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index e0a940eb0..f7237377b 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -19,21 +19,17 @@ #pragma once #include -#include #include #include #include #include -#include "paimon/catalog/identifier.h" #include "paimon/result.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" namespace paimon { -class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `WriteContext` is some configuration for write operations. @@ -43,38 +39,20 @@ class MemoryPool; /// @see WriteContextBuilder class PAIMON_EXPORT WriteContext { public: - WriteContext(const std::string& root_path, const std::string& commit_user, - bool is_streaming_mode, bool ignore_num_bucket_check, bool ignore_previous_files, + WriteContext(const std::string& commit_user, bool is_streaming_mode, + bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, const std::optional& write_id, - const std::string& branch, const std::vector& write_schema, + const std::vector& write_schema, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::string& temp_directory, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, - const std::shared_ptr& format_table, - const std::shared_ptr& catalog = nullptr, - const std::optional& identifier = std::nullopt); + const std::shared_ptr& realtime_context); ~WriteContext(); - const std::string& GetRootPath() const { - return root_path_; - } - const std::string& GetCommitUser() const { return commit_user_; } - const std::map& GetFileSystemSchemeToIdentifierMap() const { - return fs_scheme_to_identifier_map_; - } - - const std::map& GetOptions() const { - return options_; - } - bool IsStreamingMode() const { return is_streaming_mode_; } @@ -91,10 +69,6 @@ class PAIMON_EXPORT WriteContext { return write_id_; } - const std::string& GetBranch() const { - return branch_; - } - const std::vector& GetWriteSchema() const { return write_schema_; } @@ -111,10 +85,6 @@ class PAIMON_EXPORT WriteContext { return temp_directory_; } - std::shared_ptr GetSpecificFileSystem() const { - return specific_file_system_; - } - bool EnableMultiThreadSpill() const { return enable_multi_thread_spill_; } @@ -124,26 +94,8 @@ class PAIMON_EXPORT WriteContext { return realtime_context_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - - /// Returns the catalog supplying table metadata, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - private: - std::string root_path_; std::string commit_user_; - std::string branch_; bool is_streaming_mode_; bool ignore_num_bucket_check_; bool ignore_previous_files_; @@ -153,53 +105,16 @@ class PAIMON_EXPORT WriteContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::string temp_directory_; - std::shared_ptr specific_file_system_; - std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; - std::map options_; - std::shared_ptr format_table_; - std::shared_ptr catalog_; - std::optional identifier_; }; /// `WriteContextBuilder` used to build a `WriteContext`, has input validation. class PAIMON_EXPORT WriteContextBuilder { public: - /// Constructs a `WriteContextBuilder` with required parameters. - /// @param root_path The root path of the table. - /// @param commit_user The user identifier for commit operations. - WriteContextBuilder(const std::string& root_path, const std::string& commit_user); - - /// Constructs a `WriteContextBuilder` for a format table that is already loaded: the only way - /// to write one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()`, `WithCatalog()` and a branch - /// are refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to write to, as `Catalog::GetFormatTable()` hands it back. - explicit WriteContextBuilder(const std::shared_ptr& table); + explicit WriteContextBuilder(const std::string& commit_user); ~WriteContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. - /// @note The options map will clear the options added by `AddOption()` before. - /// @param options The configuration options map. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& SetOptions(const std::map& options); - - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. - /// - /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` - /// instead. - /// @param key The option key. - /// @param value The option value. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set whether to enable streaming mode (default is false) /// @return Reference to this builder for method chaining. WriteContextBuilder& WithStreamingMode(bool is_streaming_mode); @@ -237,69 +152,17 @@ class PAIMON_EXPORT WriteContextBuilder { /// @return Reference to this builder for method chaining. WriteContextBuilder& WithWriteId(int32_t write_id); - /// Write to specific branch, default is main. - /// - /// The identifier of `WithCatalog()` and the `branch` option may name the branch too; naming - /// two different ones is refused rather than silently resolved. An empty name is the main - /// branch. - /// @param branch Name of the branch to write to. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& WithBranch(const std::string& branch); - /// For data evolution, user can write partial specific fields from table schema. /// If not set, write all fields in table. /// @return Reference to this builder for method chaining. WriteContextBuilder& WithWriteSchema(const std::vector& write_schema); - /// Sets a custom file system instance to be used for all file operations in this write context. - /// This bypasses the global file system registry and uses the provided implementation directly. - /// - /// @param file_system The file system to use. - /// @return Reference to this builder for method chaining. - /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) - WriteContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - - /// Loads a native table's schema and, for catalogs with version management, its latest - /// snapshot. Use the same catalog and identifier as `CommitContextBuilder` to restore and - /// refresh real-time progress. - /// - /// A branch is addressed by the identifier, as `tbl$branch_dev`, so that the catalog answers - /// for that branch. Its schema is read from the branch directory rather than from the catalog, - /// just as a read of that branch reads it. - /// Data, manifests and historical metadata remain on the file system. The catalog supplies - /// that file system unless overridden by `WithFileSystem()` or - /// `WithFileSystemSchemeToIdentifierMap()`. - /// @param catalog Non-null catalog, kept alive while the writer uses its snapshot loader. - /// @param identifier The native table to write to. - /// @return Reference to this builder for method chaining. - WriteContextBuilder& WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier); - /// Enables the real-time write path with the provided shared context. /// @param realtime_context Non-null context that owns the real-time stores. /// @return Reference to this builder for method chaining. WriteContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Sets a mapping from URI schemes (e.g., "file", "oss") to registered file system - /// identifiers. This allows selecting different pre-registered file system implementations - /// based on the URI scheme at runtime. - /// - /// @param fs_scheme_to_identifier_map Map from URI scheme (like "oss") to the corresponding - /// file system identifier. - /// @return Reference to this builder for method chaining. - /// @note - /// - This method is intended for environments where multiple file systems are pre-registered. - /// - The specified identifiers must correspond to file systems that have been registered at - /// compile time or initialization. - /// - Cannot be used together with `WithFileSystem()`. - /// - If not set, use default file system (configured in `Options::FILE_SYSTEM`). - /// Example: - /// builder.WithFileSystemSchemeToIdentifierMap({{"oss", "jindo"}, {"file", "local"}}); - /// - WriteContextBuilder& WithFileSystemSchemeToIdentifierMap( - const std::map& fs_scheme_to_identifier_map); - /// Set the thread number for write buffer spill operations. (default is 0) /// If <= 0, threading is disabled for spill IPC read/write. /// If > 0, sets arrow CPU thread pool capacity for spill operations. diff --git a/src/paimon/core/append/append_compact_coordinator_test.cpp b/src/paimon/core/append/append_compact_coordinator_test.cpp index acdf59cb0..df11f129c 100644 --- a/src/paimon/core/append/append_compact_coordinator_test.cpp +++ b/src/paimon/core/append/append_compact_coordinator_test.cpp @@ -29,6 +29,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/path_util.h" @@ -79,11 +80,15 @@ class AppendCompactCoordinatorTest : public ::testing::Test { const std::map& partition, int32_t bucket, const std::shared_ptr& write_array, int64_t commit_identifier) { - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique( @@ -97,11 +102,15 @@ class AppendCompactCoordinatorTest : public ::testing::Test { Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 978c20e9d..dddbf3864 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -67,7 +67,8 @@ Result> Catalog::Create( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system, fs_scheme_to_identifier_map)); - return std::make_unique(core_options.GetFileSystem(), root_path, options); + return std::make_unique(core_options.GetFileSystem(), root_path, options, + fs_scheme_to_identifier_map); } bool Catalog::SupportsVersionManagement() const { diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index bb91ca88f..c59a4d9fd 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -56,8 +56,10 @@ struct ArrowSchema; namespace paimon { FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options) + const std::map& catalog_options, + const std::map& fs_scheme_map) : fs_(fs), + fs_scheme_map_(fs_scheme_map), warehouse_(warehouse), catalog_options_(catalog_options), logger_(Logger::GetLogger("FileSystemCatalog")) {} @@ -322,9 +324,10 @@ Result> FileSystemCatalog::LoadTableSchema( dynamic_options[Options::BRANCH] = branch.value(); } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, table_path, - latest_schema.value(), dynamic_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema.value(), dynamic_options, "", fs_scheme_map_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, system_table->ArrowSchema()); return std::make_shared(std::move(arrow_schema)); @@ -356,7 +359,7 @@ Result> FileSystemCatalog::LoadSystemTable( dynamic_options[Options::BRANCH] = branch; PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); return SystemTableLoader::Load(identifier, fs_, table_path, latest_schema.value(), - dynamic_options, {}); + dynamic_options, fs_scheme_map_); } Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { @@ -404,8 +407,12 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + std::shared_ptr catalog = std::const_pointer_cast(weak_from_this().lock()); + if (!catalog) { + catalog = std::make_shared(*this); + } return FileStoreTable::Create(fs_, table_path, *latest_schema, identifier, /*uuid=*/"", branch, - {}); + fs_scheme_map_, catalog); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 37bb23ea6..fba7519e0 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -42,7 +42,8 @@ class Logger; class FileSystemCatalog : public Catalog { public: FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options); + const std::map& catalog_options, + const std::map& fs_scheme_map = {}); Status CreateDatabase(const std::string& db_name, const std::map& options, @@ -110,6 +111,7 @@ class FileSystemCatalog : public Catalog { const std::vector& external_paths); std::shared_ptr fs_; + std::map fs_scheme_map_; std::string warehouse_; std::map catalog_options_; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 10dd814dd..bb6f10eca 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -26,6 +26,7 @@ #include "arrow/c/helpers.h" #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/path_util.h" @@ -1487,13 +1488,18 @@ TEST(FileSystemCatalogTest, TestClaimingVersionManagementWithoutImplementingItIs ASSERT_NOK_WITH_MSG(CheckVersionManagementImplemented(claiming), "does not implement VersionManagedCatalog"); - CommitContextBuilder claiming_builder(dir->Str(), "commit_user_1"); + TableLoadOptions claiming_builder_load_options{dir->Str()}; + std::map claiming_builder_options; + CommitContextBuilder claiming_builder("commit_user_1"); + claiming_builder_options = options; + claiming_builder_load_options.catalog = claiming; + claiming_builder_load_options.identifier.emplace(Identifier("db", "tbl")); ASSERT_OK_AND_ASSIGN(std::unique_ptr claiming_context, - claiming_builder.SetOptions(options) - .WithCatalog(claiming, Identifier("db", "tbl")) - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(claiming_context)), - "does not implement VersionManagedCatalog"); + claiming_builder.Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(claiming_builder_load_options, std::move(claiming_context), + claiming_builder_options), + "does not implement VersionManagedCatalog"); std::shared_ptr plain = std::make_shared(core_options.GetFileSystem(), dir->Str(), options); diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index 7612779ec..ff3322780 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -26,6 +26,7 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "arrow/type.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/mem_utils.h" @@ -177,16 +178,19 @@ Result> CreateBatchReader( const std::string& table_path, const std::vector& read_field_names, const std::shared_ptr& indexed_split, const CoreOptions& core_options, const std::shared_ptr& pool) { - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(core_options.ToMap()) - .WithFileSystem(core_options.GetFileSystem()) - .EnablePrefetch(true) - .WithMemoryPool(pool) - .SetReadFieldNames(read_field_names); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = core_options.ToMap(); + read_context_builder_load_options.file_system = core_options.GetFileSystem(); + read_context_builder.EnablePrefetch(true).WithMemoryPool(pool).SetReadFieldNames( + read_field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); return table_read->CreateReader(indexed_split); } diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..c2d0e3d7a 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" @@ -320,15 +321,18 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam> CreateWriter(bool with_temp_directory = true, bool write_buffer_spillable = true) const { - WriteContextBuilder builder(table_path_, kCommitUser); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(kCommitUser); std::map writer_options = options_; writer_options[Options::WRITE_BUFFER_SPILLABLE] = write_buffer_spillable ? "true" : "false"; - builder.SetOptions(writer_options).WithStreamingMode(true); + builder_options = writer_options; + builder.WithStreamingMode(true); if (with_temp_directory) { builder.WithTempDirectory(directory_->Str()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(std::move(context)); + return FileStoreWrite::Create(builder_load_options, std::move(context), builder_options); } Result>> WriteAndPrepare( @@ -354,11 +358,15 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam>& messages, int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, kCommitUser); - builder.SetOptions(options_); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(kCommitUser); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); return commit->Commit(messages, commit_identifier); } @@ -454,11 +462,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); - ScanContextBuilder scan_builder(table_path_); - scan_builder.SetPredicate(value_predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, "true"); + TableLoadOptions scan_builder_load_options{table_path_}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; + scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; int32_t data_split_count = 0; @@ -472,16 +484,19 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_GT(indexed_split_count, 0); ASSERT_EQ(0, data_split_count); - ReadContextBuilder read_builder(table_path_); + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; // The source build above keeps lazy decoding enabled. Disable it only for the final data-row // assertion, whose predicate reader is outside the maintenance path covered by this test. + read_builder_options["orc.read.enable-lazy-decoding"] = "false"; read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -686,11 +701,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); - ScanContextBuilder scan_builder(table_path_); - scan_builder.SetPredicate(value_predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, "true"); + TableLoadOptions scan_builder_load_options{table_path_}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; + scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; int32_t data_split_count = 0; @@ -704,14 +723,17 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ASSERT_EQ(0, indexed_split_count); ASSERT_GT(data_split_count, 0); - ReadContextBuilder read_builder(table_path_); + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options["orc.read.enable-lazy-decoding"] = "false"; read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -763,13 +785,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, DeletesPayloadAfterLastDefinitionI PathUtil::JoinPath(table_path_, "schema/schema-1"), schema_json, /*overwrite=*/false)); - WriteContextBuilder builder(table_path_, kCommitUser); - builder.SetOptions(evolved_options) - .WithStreamingMode(true) - .WithTempDirectory(directory_->Str()); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(kCommitUser); + builder_options = evolved_options; + builder.WithStreamingMode(true).WithTempDirectory(directory_->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN(std::vector> cleanup_messages, writer->PrepareCommit(/*wait_compaction=*/true, diff --git a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp index 1407c55b7..5fe88f8ba 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp @@ -48,18 +48,18 @@ Result> PkSortedDataFileReader::Create( const std::shared_ptr& executor, const std::shared_ptr& pool) { std::map read_options = options.ToMap(); read_options[Options::BRANCH] = branch; - ReadContextBuilder builder(root_path); - builder.SetReadFieldIds({field_id}) - .SetOptions(read_options) - .WithBranch(branch) - .WithFileSystem(options.GetFileSystem()) + ReadContextBuilder builder; + builder + .SetReadFieldIds({field_id}) + .WithExecutor(executor) .WithMemoryPool(pool) .EnablePrefetch(false) .EnablePredicateFilter(false); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shared_internal_context, - InternalReadContext::Create(read_context, table_schema, read_options)); + InternalReadContext::Create(read_context, table_schema, read_options, + options.GetFileSystem(), root_path)); return std::unique_ptr( new PkSortedDataFileReader(path_factory, shared_internal_context, pool, executor)); } diff --git a/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp index 9fc0744ef..8cf896665 100644 --- a/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/full_changelog_merge_tree_compact_rewriter.cpp @@ -65,10 +65,8 @@ FullChangelogMergeTreeCompactRewriter::Create( std::shared_ptr write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); - ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder read_context_builder; + read_context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); @@ -76,7 +74,8 @@ FullChangelogMergeTreeCompactRewriter::Create( read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + options.GetFileSystem(), path_factory_cache->RootPath())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index 390f95d84..88b355f72 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -73,10 +73,8 @@ LookupMergeTreeCompactRewriter::Create( auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); // TODO(xinyu.lxy): set executor - ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder read_context_builder; + read_context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); @@ -84,7 +82,8 @@ LookupMergeTreeCompactRewriter::Create( read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + options.GetFileSystem(), path_factory_cache->RootPath())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index 223abafed..04fe07b1c 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/path_util.h" @@ -115,9 +116,11 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { table_options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); if (!context_options.empty()) { - context_builder.SetOptions(context_options); + context_builder_options = context_options; } if (with_io_manager) { context_builder.WithTempDirectory(dir->Str()); @@ -125,7 +128,8 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(std::move(write_context)); + return FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options); } Result> CreateSingleStringFileStoreWrite( diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index afe50ddfb..7c5e2e24a 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -73,10 +73,8 @@ Result> MergeTreeCompactRewriter::Crea auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(data_schema); // TODO(xinyu.lxy): set executor - ReadContextBuilder read_context_builder(path_factory_cache->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder read_context_builder; + read_context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); @@ -84,7 +82,8 @@ Result> MergeTreeCompactRewriter::Crea read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + options.GetFileSystem(), path_factory_cache->RootPath())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, path_factory_cache->GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp index 27424a1ae..73c634ddc 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp @@ -21,6 +21,7 @@ #include "arrow/api.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" #include "paimon/core/mergetree/compact/interval_partition.h" @@ -57,10 +58,14 @@ class MergeTreeCompactRewriterTest : public testing::Test { Result>> GenerateSortedRuns( const std::string& table_path, const std::shared_ptr& table_schema, int32_t bucket, const std::map& partition) const { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetBucketFilter(bucket).SetPartitionFilter({partition}); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); auto splits = result_plan->Splits(); EXPECT_EQ(1, splits.size()); diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index 7d7aecedf..2618c2f7e 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -57,10 +57,8 @@ Result>> LookupLevels::Create( auto partition_schema = DataField::ConvertDataFieldsToArrowSchema(partition_fields); // TODO(xinyu.lxy): set executor - ReadContextBuilder read_context_builder(path_factory->RootPath()); - read_context_builder.SetOptions(options.ToMap()) - .WithFileSystem(options.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder read_context_builder; + read_context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool); @@ -68,7 +66,8 @@ Result>> LookupLevels::Create( read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_read_context, - InternalReadContext::Create(read_context, table_schema, options.ToMap())); + InternalReadContext::Create(read_context, table_schema, options.ToMap(), + options.GetFileSystem(), path_factory->RootPath())); auto split_read = std::make_unique(path_factory, internal_read_context, pool, CreateDefaultExecutor()); diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index 6e121c9fc..2f549bafe 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -30,6 +30,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/serializer/binary_row_serializer.h" @@ -599,13 +600,18 @@ TEST(AppendOnlyFileStoreScanTest, TestReconstructPredicateWithNonCastedFields) { TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); auto pool = GetDefaultPool(); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -645,12 +651,17 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -746,12 +757,16 @@ std::shared_ptr BuildScan(const std::string& table_path, const std::shared_ptr& predicate = nullptr, bool manifest_entry_lazy_decode_enabled = true, int32_t max_snapshots = 8) { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, - fmt::format("{}", max_snapshots)) - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - manifest_entry_lazy_decode_enabled ? "true" : "false") + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = + fmt::format("{}", max_snapshots); + context_builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + manifest_entry_lazy_decode_enabled ? "true" : "false"; + context_builder + .WithCache(cache); if (bucket) { context_builder.SetBucketFilter(bucket.value()); @@ -760,7 +775,9 @@ std::shared_ptr BuildScan(const std::string& table_path, context_builder.SetPredicate(predicate); } EXPECT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); - EXPECT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + EXPECT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); EXPECT_TRUE(typed_table_scan); return typed_table_scan->snapshot_reader_->scan_; diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 5887ddd0b..20b3f1968 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -366,10 +366,8 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory, const std::vector>& files, const std::optional& veto_reason) const { - ReadContextBuilder context_builder(root_path_); - context_builder.SetOptions(options_.ToMap()) - .WithFileSystem(options_.GetFileSystem()) - .EnablePrefetch(true) + ReadContextBuilder context_builder; + context_builder.EnablePrefetch(true) .SetPrefetchMaxParallelNum(1) .SetPrefetchBatchCount(3) .WithMemoryPool(pool_); @@ -393,7 +391,8 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader options[kParquetReadEnableDictionaryPassthrough] = "false"; } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_read_context, - InternalReadContext::Create(read_context, table_schema_, options)); + InternalReadContext::Create(read_context, table_schema_, options, + options_.GetFileSystem(), root_path_)); auto read = std::make_unique(file_store_path_factory_, internal_read_context, pool_, compact_executor_); diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index bfeaefced..b8c28e2cc 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -36,6 +36,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" @@ -111,11 +112,15 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -128,11 +133,15 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::map& options, const std::vector& write_schema, const std::string& json, int64_t commit_identifier) const { - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options).WithWriteSchema(write_schema); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + builder.WithWriteSchema(write_schema); EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -142,11 +151,15 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { - CommitContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(builder_load_options, + std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -225,10 +238,13 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), commit_user_); + TableLoadOptions builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); ASSERT_NOK_WITH_MSG(file_store_write->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress is only supported by a real-time writer"); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); @@ -245,11 +261,14 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - commit_user_); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -290,11 +309,16 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) { ASSERT_OK_AND_ASSIGN(std::shared_ptr batch_realtime_context, RealtimeContext::Create()); - WriteContextBuilder batch_builder(table_path, commit_user_); - batch_builder.SetOptions(options).WithRealtimeContext(batch_realtime_context); + TableLoadOptions batch_builder_load_options{table_path}; + std::map batch_builder_options; + WriteContextBuilder batch_builder(commit_user_); + batch_builder_options = options; + batch_builder.WithRealtimeContext(batch_realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_context, batch_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_writer, - FileStoreWrite::Create(std::move(batch_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_writer, + FileStoreWrite::Create(batch_builder_load_options, std::move(batch_context), + batch_builder_options)); ASSERT_NOK_WITH_MSG(batch_writer->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress requires streaming mode"); ASSERT_OK(batch_writer->Close()); @@ -302,10 +326,15 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(realtime_schema, R"([ [10, 1, "a"], @@ -378,14 +407,18 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) } TEST_F(AppendOnlyFileStoreWriteTest, TestGetMaxSequenceNumberFromMultiPartition) { - WriteContextBuilder builder( + TableLoadOptions builder_load_options; + builder_load_options.path = paimon::test::GetDataDir() + - "/orc/multi_partition_append_table.db/multi_partition_append_table/", - commit_user_); + "/orc/multi_partition_append_table.db/multi_partition_append_table/"; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options["file.format"] = "orc"; + builder_options["manifest.format"] = "orc"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr write_context, - builder.AddOption("file.format", "orc").AddOption("manifest.format", "orc").Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + auto file_store_write, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); auto write = dynamic_cast(file_store_write.get()); auto pool = GetDefaultPool(); { @@ -445,10 +478,15 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFi std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]]] ])"))); @@ -553,10 +591,15 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMultipleMapColumnsAdaptA CreateTable(dir->Str(), logical_schema, options); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - WriteContextBuilder builder(table_path, commit_user_); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]], [["c", 3], ["d", 4], ["e", 5], ["f", 6]]] ])"))); diff --git a/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp b/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp index da115ca2b..a81c2ac9a 100644 --- a/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp +++ b/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp @@ -242,15 +242,13 @@ Result> RealtimeDeduplicateCompactionChangesProvi } PAIMON_ASSIGN_OR_RAISE(result->files_to_delete_after_commit, CollectIntermediateFilesToDelete(snapshot_buckets)); - ReadContextBuilder builder(path_factory_->RootPath()); + ReadContextBuilder builder; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, - builder.SetOptions(options_.ToMap()) - .WithFileSystem(options_.GetFileSystem()) - .WithMemoryPool(pool_) - .Finish()); + builder.WithMemoryPool(pool_).Finish()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema_, options_.ToMap())); + InternalReadContext::Create(read_context, table_schema_, options_.ToMap(), + options_.GetFileSystem(), path_factory_->RootPath())); RawFileSplitRead read(path_factory_, context, pool_, executor_); const auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); const auto offset_schema = diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index e61ec950f..ce26f221c 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -19,61 +19,26 @@ #include "paimon/commit_context.h" -#include #include #include -#include "fmt/format.h" -#include "paimon/catalog/catalog.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { -CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - bool append_commit_check_conflict, +CommitContext::CommitContext(const std::string& commit_user, bool ignore_empty_commit, + bool use_rest_catalog_commit, bool append_commit_check_conflict, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) - : CommitContext(root_path, commit_user, ignore_empty_commit, use_rest_catalog_commit, - /*catalog=*/nullptr, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, - append_commit_check_conflict, memory_pool, executor, specific_file_system, - options, format_table) {} - -CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, - bool ignore_empty_commit, bool use_rest_catalog_commit, - const std::shared_ptr& catalog, - const std::optional& identifier, - const std::optional& table_id, - bool append_commit_check_conflict, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) - : root_path_(root_path), - commit_user_(commit_user), + const std::shared_ptr& executor) + : commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), use_rest_catalog_commit_(use_rest_catalog_commit), - catalog_(catalog), - identifier_(identifier), - table_id_(table_id), append_commit_check_conflict_(append_commit_check_conflict), memory_pool_(memory_pool), - executor_(executor), - specific_file_system_(specific_file_system), - options_(options), - format_table_(format_table) {} + executor_(executor) {} CommitContext::~CommitContext() = default; @@ -84,65 +49,27 @@ class CommitContextBuilder::Impl { void Reset() { ignore_empty_commit_ = true; use_rest_catalog_commit_ = false; - catalog_.reset(); - identifier_.reset(); - table_id_.reset(); append_commit_check_conflict_ = false; memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); - specific_file_system_.reset(); - options_.clear(); } private: - std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one commit to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; bool ignore_empty_commit_ = true; bool use_rest_catalog_commit_ = false; - std::shared_ptr catalog_; - std::optional identifier_; - std::optional table_id_; bool append_commit_check_conflict_ = false; std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); - std::shared_ptr specific_file_system_; - std::map options_; }; -CommitContextBuilder::CommitContextBuilder(const std::string& root_path, - const std::string& commit_user) +CommitContextBuilder::CommitContextBuilder(const std::string& commit_user) : impl_(std::make_unique()) { - impl_->root_path_ = root_path; impl_->commit_user_ = commit_user; } -CommitContextBuilder::CommitContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - CommitContextBuilder::~CommitContextBuilder() = default; -CommitContextBuilder& CommitContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -CommitContextBuilder& CommitContextBuilder::SetOptions( - const std::map& opts) { - impl_->options_ = opts; - return *this; -} - CommitContextBuilder& CommitContextBuilder::IgnoreEmptyCommit(bool ignore_empty_commit) { impl_->ignore_empty_commit_ = ignore_empty_commit; return *this; @@ -153,18 +80,6 @@ CommitContextBuilder& CommitContextBuilder::UseRESTCatalogCommit(bool use_rest_c return *this; } -CommitContextBuilder& CommitContextBuilder::WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier) { - impl_->catalog_ = catalog; - impl_->identifier_.emplace(identifier); - return *this; -} - -CommitContextBuilder& CommitContextBuilder::WithTableId(const std::string& table_id) { - impl_->table_id_ = table_id; - return *this; -} - CommitContextBuilder& CommitContextBuilder::AppendCommitCheckConflict( bool append_commit_check_conflict) { impl_->append_commit_check_conflict_ = append_commit_check_conflict; @@ -183,69 +98,10 @@ CommitContextBuilder& CommitContextBuilder::WithExecutor( return *this; } -CommitContextBuilder& CommitContextBuilder::WithFileSystem( - const std::shared_ptr& file_system) { - impl_->specific_file_system_ = file_system; - return *this; -} - Result> CommitContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot commit with null format table"); - } - if (impl_->catalog_ == nullptr && impl_->identifier_) { - return Status::Invalid("cannot commit through a null catalog"); - } - if (impl_->identifier_) { - // Before the branch is read out of the identifier: the branch of `tbl$branch_dev$options` - // reads back as `dev`, so a commit built for such a name would be aimed at another branch. - PAIMON_RETURN_NOT_OK( - CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "commit")); - } - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - impl_->identifier_, impl_->options_, - /*explicit_branch=*/std::nullopt, "commit")); - if (!BranchManager::IsMainBranch(branch)) { - // A commit which goes to a catalog, or is handed back as a request for one, names its - // branch by an object name, so the branch has to be one that reads back as itself. - // Refused here rather than at the commit, which writes metadata before publishing it. - if (impl_->catalog_ != nullptr || impl_->use_rest_catalog_commit_) { - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - } - if (impl_->catalog_ != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_.value().GetBranchName()); - if (!identifier_branch) { - PAIMON_ASSIGN_OR_RAISE(std::string table_name, - impl_->identifier_.value().GetDataTableName()); - return Status::Invalid(fmt::format( - "a commit through a catalog addresses a branch by the table identifier, so " - "name branch '{}' there as '{}$branch_{}' rather than only in the '{}' option", - branch, table_name, branch, Options::BRANCH)); - } - } - } - if (impl_->catalog_ != nullptr && impl_->use_rest_catalog_commit_) { - return Status::Invalid( - "a commit either goes to the catalog of WithCatalog() or is handed back as a request " - "for the caller to send, which is what UseRESTCatalogCommit() asks for"); - } - // The table already carries the file system it was loaded through, and from a source this - // cannot see behind, so a second answer is refused rather than silently dropped. - if (impl_->format_table_ != nullptr && impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } - PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); - if (impl_->root_path_.empty()) { - return Status::Invalid("root path is empty"); - } auto ctx = std::make_unique( - impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, - impl_->use_rest_catalog_commit_, impl_->catalog_, impl_->identifier_, impl_->table_id_, - impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_, impl_->format_table_); + impl_->commit_user_, impl_->ignore_empty_commit_, impl_->use_rest_catalog_commit_, + impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index 52e530230..6eb8752b3 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -24,6 +24,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/utils/path_util.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -34,49 +35,22 @@ namespace paimon::test { -TEST(CommitContextTest, TestLegacyConstructor) { - auto pool = GetDefaultPool(); - std::shared_ptr executor = CreateDefaultExecutor(); - auto fs = std::make_shared(); - const std::map options = {{"key", "value"}}; - CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, fs, - options, nullptr); - ASSERT_EQ(context.GetRootPath(), "table_root_path"); - ASSERT_EQ(context.GetCommitUser(), "commit_user_1"); - ASSERT_FALSE(context.IgnoreEmptyCommit()); - ASSERT_TRUE(context.UseRESTCatalogCommit()); - ASSERT_TRUE(context.AppendCommitCheckConflict()); - ASSERT_EQ(context.GetMemoryPool(), pool); - ASSERT_EQ(context.GetExecutor(), executor); - ASSERT_EQ(context.GetSpecificFileSystem(), fs); - ASSERT_EQ(context.GetOptions(), options); - ASSERT_EQ(context.GetCatalog(), nullptr); - ASSERT_FALSE(context.GetIdentifier()); - ASSERT_FALSE(context.GetTableId()); -} - TEST(CommitContextTest, TestDefaultValue) { - CommitContextBuilder builder("table_root_path", "commit_user_1"); + CommitContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); - ASSERT_EQ(ctx->GetRootPath(), expected_root_path); ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); ASSERT_TRUE(ctx->IgnoreEmptyCommit()); ASSERT_FALSE(ctx->UseRESTCatalogCommit()); - ASSERT_EQ(ctx->GetCatalog(), nullptr); - ASSERT_FALSE(ctx->GetIdentifier().has_value()); - ASSERT_EQ(ctx->GetTableId(), std::nullopt); ASSERT_FALSE(ctx->AppendCommitCheckConflict()); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); - ASSERT_TRUE(ctx->GetOptions().empty()); } TEST(CommitContextTest, TestSetContent) { - CommitContextBuilder builder("table_root_path", "commit_user_1"); + CommitContextBuilder builder("commit_user_1"); auto memory_pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); @@ -84,156 +58,18 @@ TEST(CommitContextTest, TestSetContent) { ASSERT_OK_AND_ASSIGN(auto ctx, builder.IgnoreEmptyCommit(false) .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") .AppendCommitCheckConflict(true) .WithMemoryPool(memory_pool) .WithExecutor(executor) - .WithFileSystem(fs) - .AddOption("key", "value") .Finish()); ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); - ASSERT_EQ(ctx->GetRootPath(), expected_root_path); ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); ASSERT_FALSE(ctx->IgnoreEmptyCommit()); ASSERT_TRUE(ctx->UseRESTCatalogCommit()); - ASSERT_EQ(ctx->GetTableId(), std::optional("table-uuid")); ASSERT_TRUE(ctx->AppendCommitCheckConflict()); ASSERT_EQ(ctx->GetMemoryPool(), memory_pool); ASSERT_EQ(ctx->GetExecutor(), executor); - ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); - - std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(ctx->GetOptions(), expected_options); -} - -TEST(CommitContextTest, TestWithCatalog) { - auto catalog = std::make_shared(); - CommitContextBuilder builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).WithTableId("table-uuid").Finish()); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_TRUE(ctx->GetIdentifier().has_value()); - ASSERT_EQ(ctx->GetIdentifier().value(), Identifier("db1", "t1")); - ASSERT_EQ(ctx->GetTableId(), std::optional("table-uuid")); - - CommitContextBuilder null_catalog_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(null_catalog_builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot commit through a null catalog"); - - CommitContextBuilder both_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(both_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .UseRESTCatalogCommit(true) - .Finish(), - "either goes to the catalog"); - - ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_EQ(next_ctx->GetCatalog(), nullptr); - ASSERT_FALSE(next_ctx->GetIdentifier().has_value()); - ASSERT_EQ(next_ctx->GetTableId(), std::nullopt); -} - -TEST(CommitContextTest, TestCatalogAddressesBranchByIdentifier) { - auto catalog = std::make_shared(); - - CommitContextBuilder branch_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto branch_ctx, - branch_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); - ASSERT_EQ(branch_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); - CommitContextBuilder agreeing_builder("table_root_path", "commit_user_1"); - ASSERT_OK(agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - CommitContextBuilder constructed_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto constructed_ctx, - constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_EQ(constructed_ctx->GetIdentifier().value(), Identifier("db1", "t1$branch_dev")); - CommitContextBuilder main_builder("table_root_path", "commit_user_1"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .AddOption(Options::BRANCH, "main") - .Finish()); - - CommitContextBuilder mixed_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .AddOption(Options::BRANCH, "main") - .Finish(), - "but both 'dev' and 'main' were named"); - CommitContextBuilder mixed_option_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - mixed_option_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .AddOption(Options::BRANCH, "dev") - .Finish(), - "but both 'main' and 'dev' were named"); - - CommitContextBuilder option_only_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(option_only_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev") - .Finish(), - "name branch 'dev' there as 't1$branch_dev'"); - - CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK(upper_main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_MAIN")) - .AddOption(Options::BRANCH, "main") - .Finish()); - - CommitContextBuilder upper_main_option_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(upper_main_option_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "MAIN") - .Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - CommitContextBuilder splitter_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(splitter_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev$options") - .Finish(), - "a branch a catalog addresses cannot contain '$'"); - - CommitContextBuilder system_table_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - system_table_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev$options")).Finish(), - "Cannot 'commit' for system table"); -} - -TEST(CommitContextTest, TestBranch) { - CommitContextBuilder file_system_builder("table_root_path", "commit_user_1"); - ASSERT_OK(file_system_builder.AddOption(Options::BRANCH, "dev").Finish()); - - CommitContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, - upper_main_builder.AddOption(Options::BRANCH, "MAIN").Finish()); - ASSERT_EQ(upper_main_ctx->GetOptions().at(Options::BRANCH), "MAIN"); - CommitContextBuilder splitter_builder("table_root_path", "commit_user_1"); - ASSERT_OK(splitter_builder.AddOption(Options::BRANCH, "dev$options").Finish()); - - CommitContextBuilder rest_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto rest_ctx, - rest_builder.UseRESTCatalogCommit(true).AddOption(Options::BRANCH, "dev").Finish()); - ASSERT_TRUE(rest_ctx->UseRESTCatalogCommit()); - ASSERT_EQ(rest_ctx->GetOptions().at(Options::BRANCH), "dev"); - CommitContextBuilder rest_upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(rest_upper_main_builder.UseRESTCatalogCommit(true) - .AddOption(Options::BRANCH, "MAIN") - .Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - CommitContextBuilder escaping_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(escaping_builder.AddOption(Options::BRANCH, "dev/../../outside").Finish(), - "branch name cannot contain path separators"); -} - -TEST(CommitContextTest, TestSetOptionsOverridesAddedOptions) { - CommitContextBuilder builder("table_root_path", "commit_user_1"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(ctx->GetOptions(), expected_options); } } // namespace paimon::test diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index 69607ef25..02dbb1ddd 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -29,6 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/file_index/bitmap/bitmap_file_index.h" @@ -411,10 +412,16 @@ TEST_F(DataEvolutionSplitReadTest, TestBlobFallbackValidatesRangesBeforeOpeningF {Options::DATA_EVOLUTION_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - ReadContextBuilder builder(dir->Str()); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(/*path_factory=*/nullptr, context, GetDefaultPool(), /*executor=*/nullptr); auto blob = @@ -449,10 +456,16 @@ TEST_F(DataEvolutionSplitReadTest, TestEmptyBlobReaders) { {Options::DATA_EVOLUTION_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - ReadContextBuilder builder(dir->Str()); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, FileStorePathFactory::Create(dir->Str(), schema, {}, "default", "parquet", "data-", @@ -1108,11 +1121,18 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { std::vector{"_ROW_ID", "_SEQUENCE_NUMBER"}}) { auto counting_fs = std::make_shared( PathUtil::JoinPath(bucket_path, main->file_name)); - ReadContextBuilder builder(dir->Str()); - builder.WithFileSystem(counting_fs).EnablePrefetch(false).SetReadFieldNames(projection); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.file_system = counting_fs; + builder.EnablePrefetch(false).SetReadFieldNames(projection); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); auto split_read = std::make_unique(path_factory, context, GetDefaultPool(), executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, split_read->CreateReader(split)); @@ -1133,12 +1153,19 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { // A selection confined to the first VECTOR file must not include rows from the second. { - ReadContextBuilder builder(dir->Str()); - builder.WithFileSystem(fs).EnablePrefetch(false).SetReadFieldNames( + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.file_system = fs; + builder.EnablePrefetch(false).SetReadFieldNames( {"id", "embedding", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(path_factory, context, GetDefaultPool(), executor); auto selected_split = std::make_shared(split, std::vector{Range(0, 1)}); @@ -1173,12 +1200,19 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { {vector1, legacy_main, vector0}); ASSERT_OK_AND_ASSIGN(std::shared_ptr legacy_split, legacy_split_builder.RawConvertible(false).Build()); - ReadContextBuilder builder(dir->Str()); - builder.WithFileSystem(fs).EnablePrefetch(false).SetReadFieldNames( + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.file_system = fs; + builder.EnablePrefetch(false).SetReadFieldNames( {"id", "embedding", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, evolved_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, evolved_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(path_factory, context, GetDefaultPool(), executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, split_read.CreateReader(legacy_split)); @@ -1394,10 +1428,16 @@ TEST_F(DataEvolutionSplitReadTest, TestFileIndexSelectionOnlyPushesDownNormalGro {Options::FILE_INDEX_READ_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - ReadContextBuilder builder(dir->Str()); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(/*path_factory=*/nullptr, context, pool, /*executor=*/nullptr); @@ -1568,10 +1608,16 @@ TEST_F(DataEvolutionSplitReadTest, TestGroupBitmapCombinesWithRowRangesAndDvInFi {Options::DATA_EVOLUTION_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - ReadContextBuilder builder(dir->Str()); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); DataEvolutionSplitRead split_read(/*path_factory=*/nullptr, context, GetDefaultPool(), /*executor=*/nullptr); split_read.file_indexes_evaluated_ = true; diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index be1482052..211a7fea9 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -37,6 +37,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/data/binary_row.h" @@ -280,8 +281,8 @@ class FileStoreCommitImplTest : public testing::Test { } std::shared_ptr CreateCatalogServingSchemaId( - int64_t schema_id) const { - std::shared_ptr catalog = CreateCatalog(); + int64_t schema_id, const std::string& table_uuid = "mock-table-uuid") const { + std::shared_ptr catalog = CreateCatalog(true, table_uuid); catalog->SetTableSchema(TableSchemaWithId(schema_id)); return catalog; } @@ -324,11 +325,14 @@ class FileStoreCommitImplTest : public testing::Test { } void CommitAsCompetitor(int32_t round, int64_t commit_identifier) { - CommitContextBuilder builder(table_path_, "competitor"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr competitor, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("competitor"); + builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr competitor, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(competitor->Commit(CommitMessagesOfRound(round), commit_identifier)); } @@ -344,19 +348,22 @@ class FileStoreCommitImplTest : public testing::Test { Result> CreateCatalogCommit( const CatalogCommitSpec& spec) const { - CommitContextBuilder builder(spec.root_path.empty() ? table_path_ : spec.root_path, - spec.commit_user); - builder.AddOption(Options::FILE_SYSTEM, "local") - .WithCatalog(spec.catalog, spec.identifier.value()) - .AppendCommitCheckConflict(spec.append_commit_check_conflict); + TableLoadOptions builder_load_options{spec.root_path.empty() ? table_path_ + : spec.root_path}; + std::map builder_options; + CommitContextBuilder builder(spec.commit_user); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_load_options.catalog = spec.catalog; + builder_load_options.identifier.emplace(spec.identifier.value()); + builder.AppendCommitCheckConflict(spec.append_commit_check_conflict); if (spec.table_id) { - builder.WithTableId(spec.table_id.value()); + builder_load_options.table_id = spec.table_id.value(); } for (const auto& [key, value] : spec.options) { - builder.AddOption(key, value); + builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(builder_load_options, std::move(context), builder_options); } Result> CreateCatalogCommit( @@ -413,16 +420,20 @@ class FileStoreCommitImplTest : public testing::Test { void CheckExpireIsRefusedWithoutDeleting(const std::string& branch, const std::string& expected_msg) { - CommitContextBuilder builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, branch) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = branch; + builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(PrepareFirstCommitMessageFiles()); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK(commit->DropPartition({{{"f1", "10"}}}, /*commit_identifier=*/2)); @@ -611,13 +622,16 @@ class FileStoreCommitImplTest : public testing::Test { }; TEST_F(FileStoreCommitImplTest, TestCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -639,15 +653,18 @@ TEST_F(FileStoreCommitImplTest, TestCommit) { TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TimezoneGuard guard("Asia/Shanghai"); - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_load_options.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); + context_builder.UseRESTCatalogCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_NOK_WITH_MSG(commit->GetLastCommitTableRequest(), @@ -697,10 +714,14 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { std::vector> msgs = CommitMessagesOfRound(1); - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); ASSERT_OK(base_commit->Commit(msgs, 1)); std::string snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"); @@ -708,13 +729,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { ASSERT_TRUE(base_snapshot.Uuid().has_value()); const std::string base_uuid = base_snapshot.Uuid().value(); - CommitContextBuilder rest_builder(table_path_, "commit_user_1"); + TableLoadOptions rest_builder_load_options{table_path_}; + std::map rest_builder_options; + CommitContextBuilder rest_builder("commit_user_1"); + rest_builder_options[Options::FILE_SYSTEM] = "local"; + rest_builder_load_options.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, - rest_builder.AddOption(Options::FILE_SYSTEM, "local") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto rest_commit, FileStoreCommit::Create(std::move(rest_context))); + rest_builder.UseRESTCatalogCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN(auto rest_commit, + FileStoreCommit::Create(rest_builder_load_options, std::move(rest_context), + rest_builder_options)); ASSERT_OK(rest_commit->Commit(msgs, 2)); ASSERT_OK_AND_ASSIGN(std::string request_str, rest_commit->GetLastCommitTableRequest()); @@ -739,7 +763,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { } TEST_F(FileStoreCommitImplTest, TestCatalogCommitTakesTheSnapshot) { - auto catalog = CreateCatalog(); + auto catalog = CreateCatalog(true, "table-uuid-1"); CatalogCommitSpec spec; spec.catalog = catalog; spec.table_id = "table-uuid-1"; @@ -754,7 +778,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitTakesTheSnapshot) { ASSERT_EQ(call.table_uuid, std::optional("table-uuid-1")); ASSERT_EQ(call.base_snapshot_uuid, std::nullopt); ASSERT_EQ(call.snapshot.Id(), 1); - ASSERT_EQ(catalog->GetTableCalls(), 0u); + ASSERT_EQ(catalog->GetTableCalls(), 1u); ASSERT_GT(catalog->LoadTableSchemaCalls(), 0u); ASSERT_OK_AND_ASSIGN(std::string request_str, commit->GetLastCommitTableRequest()); @@ -774,25 +798,30 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitTakesTheSnapshot) { } TEST_F(FileStoreCommitImplTest, TestCommitToBranch) { - CommitContextBuilder missing_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_context, - missing_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(missing_context)), - "not found latest schema in branch dev"); + TableLoadOptions missing_builder_load_options{table_path_}; + std::map missing_builder_options; + CommitContextBuilder missing_builder("commit_user_1"); + missing_builder_options[Options::FILE_SYSTEM] = "local"; + missing_builder_options[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_context, missing_builder.Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(missing_builder_load_options, std::move(missing_context), + missing_builder_options), + "not found latest schema in branch dev"); SchemaManager branch_schema_manager(file_system_, table_path_, "dev"); ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - CommitContextBuilder builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); std::string branch_snapshots = PathUtil::JoinPath(table_path_, "branch/branch-dev/snapshot"); @@ -818,15 +847,17 @@ TEST_F(FileStoreCommitImplTest, TestRequestOnlyCommitToBranch) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - CommitContextBuilder builder(table_path_, "commit_user_1"); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::BRANCH] = "dev"; + builder_load_options.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + builder.UseRESTCatalogCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK_AND_ASSIGN(std::string request_str, commit->GetLastCommitTableRequest()); @@ -937,13 +968,15 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitToBranchTakesTheBranchSnapshot) ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BRANCH, "dev") - .Finish()); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + base_builder_options[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr base_commit, - FileStoreCommit::Create(std::move(base_context))); + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); SnapshotManager branch_snapshot_manager(file_system_, table_path_, "dev"); ASSERT_OK_AND_ASSIGN(std::optional base_snapshot, @@ -952,7 +985,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitToBranchTakesTheBranchSnapshot) ASSERT_EQ(base_snapshot.value().Id(), 1); ASSERT_TRUE(base_snapshot.value().Uuid().has_value()); - auto catalog = CreateCatalog(); + auto catalog = CreateCatalog(true, "table-uuid-1"); catalog->SetHeldSnapshot(base_snapshot.value()); std::shared_ptr log = InstallCommitLock(catalog.get()); CatalogCommitSpec spec; @@ -989,7 +1022,8 @@ TEST_F(FileStoreCommitImplTest, TestBranchCommitReadsTheSchemaFromTheBranch) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - std::shared_ptr catalog = CreateCatalogServingSchemaId(7); + std::shared_ptr catalog = + CreateCatalogServingSchemaId(7, "table-uuid-1"); CatalogCommitSpec spec; spec.catalog = catalog; spec.identifier.emplace("db", "tbl$branch_dev"); @@ -1002,7 +1036,7 @@ TEST_F(FileStoreCommitImplTest, TestBranchCommitReadsTheSchemaFromTheBranch) { ASSERT_EQ(catalog->CommitCalls().front().snapshot.SchemaId(), 0); ASSERT_EQ(catalog->LoadTableSchemaCalls(), 0u); - catalog->SetTableSchema(nullptr); + catalog->SetTableSchema(TableSchemaWithId(99)); ASSERT_OK_AND_ASSIGN(std::unique_ptr next_commit, CreateCatalogCommit(spec)); ASSERT_OK(next_commit->Commit(CommitMessagesOfRound(2), /*commit_identifier=*/2)); @@ -1016,7 +1050,7 @@ TEST_F(FileStoreCommitImplTest, TestBranchCommitNamesTheBranchWhenItFails) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - auto catalog = CreateCatalog(Status::IOError("catalog unreachable")); + auto catalog = CreateCatalog(Status::IOError("catalog unreachable"), "table-uuid"); CatalogCommitSpec spec; spec.catalog = catalog; spec.identifier.emplace("db", "tbl$branch_dev"); @@ -1029,7 +1063,7 @@ TEST_F(FileStoreCommitImplTest, TestBranchCommitNamesTheBranchWhenItFails) { } TEST_F(FileStoreCommitImplTest, TestCatalogCommitCanonicalizesTheMainBranchIdentifier) { - auto catalog = CreateCatalog(); + auto catalog = CreateCatalog(true, "table-uuid-1"); CatalogCommitSpec spec; spec.catalog = catalog; spec.identifier.emplace("db", "tbl$branch_main"); @@ -1050,7 +1084,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitCanonicalizesTheMainBranchIdent } TEST_F(FileStoreCommitImplTest, TestCommitIsRefusedForATableRecreatedUnderTheSameName) { - auto catalog = CreateCatalog(true, "table-uuid-of-the-new-table"); + auto catalog = CreateCatalog(true, "table-uuid-of-the-dropped-table"); catalog->CheckTableUuid(); CatalogCommitSpec spec; spec.catalog = catalog; @@ -1058,6 +1092,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitIsRefusedForATableRecreatedUnderTheSam spec.options = {{Options::COMMIT_MAX_RETRIES, "2"}, {Options::COMMIT_MIN_RETRY_WAIT, "1ms"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, CreateCatalogCommit(spec)); + catalog->SetTableUuid("table-uuid-of-the-new-table"); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_NOK_WITH_MSG(commit->Commit(msgs), "no table with id table-uuid-of-the-dropped-table"); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1069,7 +1104,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitIsRefusedForATableRecreatedUnderTheSam ASSERT_OK(unchecked_commit->Commit(msgs, 2)); ASSERT_EQ(catalog->CommitCalls().size(), 2u); ASSERT_EQ(catalog->CommitCalls().back().table_uuid, std::nullopt); - ASSERT_EQ(catalog->GetTableCalls(), 0u); + ASSERT_EQ(catalog->GetTableCalls(), 1u); ASSERT_OK_AND_ASSIGN(std::string request_str, unchecked_commit->GetLastCommitTableRequest()); ASSERT_OK_AND_ASSIGN(CommitTableRequest request, CommitTableRequest::FromJsonString(request_str)); @@ -1112,10 +1147,14 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWinsOverTheOneUnderTheTablePath ASSERT_EQ(catalog->CommitCalls().size(), 1u); ASSERT_EQ(catalog->CommitCalls().front().snapshot.SchemaId(), 7); - CommitContextBuilder path_builder(table_path_, "commit_user_2"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, - path_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto path_commit, FileStoreCommit::Create(std::move(path_context))); + TableLoadOptions path_builder_load_options{table_path_}; + std::map path_builder_options; + CommitContextBuilder path_builder("commit_user_2"); + path_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, path_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto path_commit, + FileStoreCommit::Create(path_builder_load_options, std::move(path_context), + path_builder_options)); std::vector> other_msgs = CommitMessagesOfRound(2); ASSERT_OK(path_commit->Commit(other_msgs, 2)); ASSERT_OK_AND_ASSIGN( @@ -1182,15 +1221,17 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWhichIsNotADataTableSchemaIsRef } TEST_F(FileStoreCommitImplTest, TestTableIdWithNothingToSendItInIsRefused) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local") - .WithTableId("table-uuid") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), - "publishes the snapshot by writing it to the table directory"); - - auto catalog = CreateCatalog(); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_load_options.table_id = "table-uuid"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options), + "a table id requires a catalog or an external REST commit request"); + + auto catalog = CreateCatalog(true, "table-uuid"); catalog->SetSupportsVersionManagement(false); CatalogCommitSpec spec; spec.catalog = catalog; @@ -1216,7 +1257,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogWithoutVersionManagementWritesTheSnap TEST_F(FileStoreCommitImplTest, TestCatalogCommitRetriesWhenRefused) { for (int32_t max_retries : {0, 2}) { SCOPED_TRACE(max_retries); - auto catalog = CreateCatalog(false); + auto catalog = CreateCatalog(false, "table-uuid-2"); CatalogCommitSpec spec; spec.catalog = catalog; spec.table_id = "table-uuid-2"; @@ -1251,7 +1292,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitRetriesWhenRefused) { } TEST_F(FileStoreCommitImplTest, TestCatalogCommitHoldsALockScopePerAttempt) { - auto catalog = CreateCatalog(false); + auto catalog = CreateCatalog(false, "table-uuid-2"); std::shared_ptr log = InstallCommitLock(catalog.get()); CatalogCommitSpec spec; spec.catalog = catalog; @@ -1359,18 +1400,21 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithoutAUsableLockIsNotCreated) { ASSERT_NOK_WITH_MSG(CreateCatalogCommit(catalog), "no commit lease for this table"); ASSERT_TRUE(catalog->CommitCalls().empty()); - CommitContextBuilder builder(table_path_, "commit_user_1"); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::REST_COMMIT_ACQUIRE_LOCK] = "true"; + builder_load_options.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::REST_COMMIT_ACQUIRE_LOCK, "true") - .UseRESTCatalogCommit(true) - .WithTableId("table-uuid") - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(context)), "cannot hold a commit lease"); + builder.UseRESTCatalogCommit(true).Finish()); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options), + "cannot hold a commit lease"); } TEST_F(FileStoreCommitImplTest, TestUnknownCatalogOutcomeKeepsTheManifests) { - auto catalog = CreateCatalog(Status::IOError("catalog unreachable")); + auto catalog = CreateCatalog(Status::IOError("catalog unreachable"), "table-uuid"); CatalogCommitSpec spec; spec.catalog = catalog; spec.table_id = "table-uuid"; @@ -1406,7 +1450,7 @@ TEST_F(FileStoreCommitImplTest, TestUnknownCatalogOutcomeKeepsTheManifests) { TEST_F(FileStoreCommitImplTest, TestCatalogCommitRebasesOnTheWinnerOfEachRace) { std::vector> msgs = CommitMessagesOfRound(1); - auto catalog = CreateCatalog(false); + auto catalog = CreateCatalog(false, "table-uuid-3"); int32_t competitor_round = 0; catalog->SetOnCommit([&]() { ++competitor_round; @@ -1662,10 +1706,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitReadsThroughTheCatalogsFileSystem) { auto catalog = CreateCatalog(); catalog->SetFileSystem(fs); - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_load_options.catalog = catalog; + context_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1698,12 +1747,16 @@ TEST_F(FileStoreCommitImplTest, TestAGivenFileSystemWinsOverTheCatalogs) { auto catalog = CreateCatalog(); catalog->SetFileSystem(catalog_fs); - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.WithCatalog(catalog, Identifier("db", "tbl")) - .WithFileSystem(given_fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_load_options.catalog = catalog; + context_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + context_builder_load_options.file_system = given_fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -1975,7 +2028,7 @@ TEST_F(FileStoreCommitImplTest, TestRowIdConflictChecksSnapshotBoundsAndHistory) } TEST_F(FileStoreCommitImplTest, TestCatalogSnapshotLoaderFailurePropagates) { - auto catalog = CreateCatalog(); + auto catalog = CreateCatalog(true, "table-uuid"); catalog->SetLoadSnapshotStatus( Status::IOError("catalog unreachable while loading the snapshot")); CatalogCommitSpec spec; @@ -1989,14 +2042,19 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSnapshotLoaderFailurePropagates) { } TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::WRITE_SEQUENCE_NUMBER_INIT_MODE] = "snapshot"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + .Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -2036,16 +2094,21 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); - CommitContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::COMMIT_MAX_RETRIES, "10") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::COMMIT_MAX_RETRIES] = "10"; + context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; + context_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); @@ -2077,15 +2140,20 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); - CommitContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; + context_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); EXPECT_CALL(*mock_fs, ReadFile(testing::_, testing::_)) @@ -2149,17 +2217,22 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); - CommitContextBuilder realtime_builder(table_path_, "realtime_commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_context, - realtime_builder.AddOption(Options::REALTIME_ENABLED, "true") - .AddOption(Options::BUCKET, "2") - .AddOption(Options::COMMIT_MAX_RETRIES, "1") - .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") - .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_commit, - FileStoreCommit::Create(std::move(realtime_context))); + TableLoadOptions realtime_builder_load_options{table_path_}; + std::map realtime_builder_options; + CommitContextBuilder realtime_builder("realtime_commit_user"); + realtime_builder_options[Options::REALTIME_ENABLED] = "true"; + realtime_builder_options[Options::BUCKET] = "2"; + realtime_builder_options[Options::COMMIT_MAX_RETRIES] = "1"; + realtime_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; + realtime_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; + realtime_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_context, realtime_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr realtime_commit, + FileStoreCommit::Create(realtime_builder_load_options, std::move(realtime_context), + realtime_builder_options)); auto realtime_commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(realtime_commit))); @@ -2176,11 +2249,16 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) OffsetRange(/*begin=*/0, /*end=*/1)}); } - CommitContextBuilder concurrent_builder(table_path_, "concurrent_commit_user"); + TableLoadOptions concurrent_builder_load_options{table_path_}; + std::map concurrent_builder_options; + CommitContextBuilder concurrent_builder("concurrent_commit_user"); + concurrent_builder_load_options.file_system = file_system_; ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_context, - concurrent_builder.WithFileSystem(file_system_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_commit, - FileStoreCommit::Create(std::move(concurrent_context))); + concurrent_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr concurrent_commit, + FileStoreCommit::Create(concurrent_builder_load_options, std::move(concurrent_context), + concurrent_builder_options)); auto* mock_fs = dynamic_cast(fs.get()); ASSERT_NE(nullptr, mock_fs); @@ -2224,13 +2302,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); - CommitContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::string new_snapshot_6 = PathUtil::JoinPath(table_path, "snapshot/snapshot-6"); auto* mock_fs = dynamic_cast(fs.get()); EXPECT_CALL(*mock_fs, AtomicStore(testing::StrEq(new_snapshot_6), testing::_)) @@ -2251,13 +2332,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua bool exist, file_system_->Exists(PathUtil::JoinPath(table_path, "snapshot/snapshot-6"))); ASSERT_TRUE(exist); - CommitContextBuilder context_builder_2(table_path, "commit_user_1"); + TableLoadOptions context_builder_2_load_options{table_path}; + std::map context_builder_2_options; + CommitContextBuilder context_builder_2("commit_user_1"); + context_builder_2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_2_load_options.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context_2, - context_builder_2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); + context_builder_2.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(std::move(commit_context_2))); + ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(context_builder_2_load_options, + std::move(commit_context_2), + context_builder_2_options)); ASSERT_OK_AND_ASSIGN(int32_t num_committed, commit_2->FilterAndCommit({{1, msgs}})); ASSERT_EQ(0, num_committed); std::string new_snapshot_7 = PathUtil::JoinPath(table_path, "snapshot/snapshot-7"); @@ -2275,14 +2360,19 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua } TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "5kb") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "5kb"; + context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); { std::vector> msgs = @@ -2338,13 +2428,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { } TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); { std::vector> msgs = @@ -2409,12 +2502,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2466,10 +2562,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2493,10 +2593,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2525,10 +2629,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTablePath) { - CommitContextBuilder base_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, - base_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto base_commit, FileStoreCommit::Create(std::move(base_context))); + TableLoadOptions base_builder_load_options{table_path_}; + std::map base_builder_options; + CommitContextBuilder base_builder("commit_user_1"); + base_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto base_commit, + FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); } @@ -2542,23 +2650,29 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTabl } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->RollbackToAsLatest(/*target_snapshot_id=*/1), "Latest snapshot is null, can not roll back."); } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", @@ -2587,14 +2701,19 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) /*ignore_if_exists=*/false)); std::string dv_table_path = PathUtil::JoinPath(test_root_, "foo.db/dv_bar"); - CommitContextBuilder context_builder(dv_table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{dv_table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2664,12 +2783,15 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF // before writing. A stateful Exists() mock reproduces exactly that ordering. ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(fs) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2706,12 +2828,15 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF } TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); { std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -2751,12 +2876,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { } TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -2808,13 +2936,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { } TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); { std::vector> msgs = @@ -2926,14 +3057,17 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs; ASSERT_OK(commit->Commit(msgs)); std::shared_ptr metrics = commit->GetCommitMetrics(); @@ -2944,14 +3078,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { } TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -2974,14 +3111,17 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); std::vector changes; @@ -3008,14 +3148,17 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3035,14 +3178,17 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); std::vector changes; @@ -3367,18 +3513,23 @@ TEST_F(FileStoreCommitImplTest, TestCatalogExpireWithoutPublishedSnapshots) { } TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_EXPIRE_LIMIT, "30") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .IgnoreEmptyCommit(true) - .Finish()); - - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + context_builder_options[Options::SNAPSHOT_EXPIRE_LIMIT] = "30"; + context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3439,18 +3590,23 @@ TEST_F(FileStoreCommitImplTest, TestExpireOnMainOfTableWithBranchesIsNotSupporte } TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1") - .AddOption(Options::SNAPSHOT_EXPIRE_LIMIT, "30") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .IgnoreEmptyCommit(true) - .Finish()); - - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1"; + context_builder_options[Options::SNAPSHOT_EXPIRE_LIMIT] = "30"; + context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3486,14 +3642,17 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { } TEST_F(FileStoreCommitImplTest, TestTruncateTable) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3519,10 +3678,14 @@ TEST_F(FileStoreCommitImplTest, TestTruncateTable) { } TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3571,10 +3734,14 @@ TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { } TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3595,10 +3762,14 @@ TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) { // A delete that fails with an IO error (e.g. permission denied) must be swallowed by // best-effort cleanup, mirroring Java deleteQuietly: Abort still returns OK and does not // propagate the failure. - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3634,13 +3805,16 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { // Truncating a table that has never been committed succeeds and produces an OVERWRITE snapshot // with no added and no deleted files: there is nothing to overwrite, but the overwrite is still // materialized. - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.IgnoreEmptyCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_OK(commit->TruncateTable(/*commit_identifier=*/1)); @@ -3659,14 +3833,17 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { } TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector> msgs; @@ -3679,14 +3856,19 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { } TEST_F(FileStoreCommitImplTest, TestCollectChanges) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::BUCKET, "10") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::BUCKET] = "10"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -3722,13 +3904,16 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { } TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3753,13 +3938,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3793,13 +3981,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3824,13 +4015,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) } TEST_F(FileStoreCommitImplTest, FilterAndCommit) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector data_files = { @@ -3871,13 +4065,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommit) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3893,13 +4090,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3927,13 +4127,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3960,13 +4163,16 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { } TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3999,13 +4205,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4030,15 +4239,20 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::ROW_TRACKING_ENABLED, "true") - .AddOption(Options::DATA_EVOLUTION_ENABLED, "true") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::ROW_TRACKING_ENABLED] = "true"; + context_builder_options[Options::DATA_EVOLUTION_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4062,14 +4276,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4095,13 +4312,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4133,14 +4353,17 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); + context_builder.IgnoreEmptyCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4173,13 +4396,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4207,13 +4433,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); std::vector data_files = { @@ -4262,13 +4491,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4301,13 +4533,16 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4335,13 +4570,16 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4365,14 +4603,19 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { } TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true") - .Finish()); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::COMMIT_DISCARD_DUPLICATE_FILES] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4397,12 +4640,15 @@ TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -4429,12 +4675,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - CommitContextBuilder context_builder2(tmp_table_path, "commit_user_1"); + TableLoadOptions context_builder2_load_options{tmp_table_path}; + std::map context_builder2_options; + CommitContextBuilder context_builder2("commit_user_1"); + context_builder2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder2_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context2, - context_builder2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - auto commit2 = FileStoreCommit::Create(std::move(commit_context2)); + context_builder2.Finish()); + auto commit2 = FileStoreCommit::Create( + context_builder2_load_options, std::move(commit_context2), context_builder2_options); CHECK_HOOK_STATUS(commit2.status(), i); CHECK_HOOK_STATUS(commit2.value()->Commit(msgs), i); commit_run_complete = true; @@ -4458,9 +4707,12 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_TRUE(is_oss); // REST commit with local path should work without the object store flag - CommitContextBuilder builder(table_path_, "commit_user_1"); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.UseRESTCatalogCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(builder_load_options, std::move(ctx), + builder_options)); std::vector> msgs = CommitMessagesOfRound(1); ASSERT_OK(commit->Commit(msgs)); @@ -4468,10 +4720,15 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_FALSE(json.empty()); auto catalog = CreateCatalog(); - CommitContextBuilder catalog_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto catalog_ctx, - catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK_AND_ASSIGN(auto catalog_commit, FileStoreCommit::Create(std::move(catalog_ctx))); + TableLoadOptions catalog_builder_load_options{table_path_}; + std::map catalog_builder_options; + CommitContextBuilder catalog_builder("commit_user_1"); + catalog_builder_load_options.catalog = catalog; + catalog_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + ASSERT_OK_AND_ASSIGN(auto catalog_ctx, catalog_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto catalog_commit, + FileStoreCommit::Create(catalog_builder_load_options, + std::move(catalog_ctx), catalog_builder_options)); ASSERT_OK(catalog_commit->Commit(msgs)); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -4480,15 +4737,22 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_TRUE(path_is_object_store); auto object_store_catalog = CreateCatalog(); object_store_catalog->SetFileSystem(file_system_); - CommitContextBuilder object_store_builder(object_store_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto object_store_ctx, - object_store_builder.WithCatalog(object_store_catalog, Identifier("db", "tbl")).Finish()); - ASSERT_OK(FileStoreCommit::Create(std::move(object_store_ctx))); - - CommitContextBuilder plain_builder(object_store_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto plain_ctx, plain_builder.WithFileSystem(file_system_).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(plain_ctx)), + TableLoadOptions object_store_builder_load_options{object_store_path}; + std::map object_store_builder_options; + CommitContextBuilder object_store_builder("commit_user_1"); + object_store_builder_load_options.catalog = object_store_catalog; + object_store_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + ASSERT_OK_AND_ASSIGN(auto object_store_ctx, object_store_builder.Finish()); + ASSERT_OK(FileStoreCommit::Create(object_store_builder_load_options, + std::move(object_store_ctx), object_store_builder_options)); + + TableLoadOptions plain_builder_load_options{object_store_path}; + std::map plain_builder_options; + CommitContextBuilder plain_builder("commit_user_1"); + plain_builder_load_options.file_system = file_system_; + ASSERT_OK_AND_ASSIGN(auto plain_ctx, plain_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(plain_builder_load_options, std::move(plain_ctx), + plain_builder_options), "invalid scheme oss for local file system"); } @@ -4510,10 +4774,15 @@ TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) { std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - CommitContextBuilder builder(pk_table_path, "test_user"); - builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); + TableLoadOptions builder_load_options{pk_table_path}; + std::map builder_options; + CommitContextBuilder builder("test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto committer, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + auto committer, + FileStoreCommit::Create(builder_load_options, std::move(commit_context), builder_options)); ASSERT_TRUE(committer != nullptr); } @@ -4543,14 +4812,19 @@ TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsAllowsManifestDeleteFileDro } TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::MANIFEST_DELETE_FILE_DROP_STATS] = "true"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(context_builder_load_options, std::move(commit_context), + context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4572,14 +4846,19 @@ TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { } TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::MANIFEST_DELETE_FILE_DROP_STATS] = "true"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(context_builder_load_options, std::move(commit_context), + context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4647,21 +4926,28 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { } TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->DropPartition({}, /*commit_identifier=*/1), "partitions list cannot be empty"); } TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4695,10 +4981,14 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4709,10 +4999,14 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4736,13 +5030,18 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { } TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::DYNAMIC_PARTITION_OVERWRITE, "false") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::DYNAMIC_PARTITION_OVERWRITE] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4764,12 +5063,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) } TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); const BinaryRow partition = CreateIntRow(10); @@ -4809,12 +5111,15 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - CommitContextBuilder builder(pk_table_path, "test_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions builder_load_options{pk_table_path}; + std::map builder_options; + CommitContextBuilder builder("test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::OVERWRITE_UPGRADE] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, + FileStoreCommit::Create(builder_load_options, std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4859,13 +5164,16 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) } TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AppendCommitCheckConflict(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.AppendCommitCheckConflict(true).Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -4882,12 +5190,16 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // First snapshot is committed in the default (scan) mode, so it does NOT carry the // max-sequence-number property. { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), + context_builder_options)); std::vector> msgs1 = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", @@ -4897,13 +5209,18 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // Second commit uses snapshot-init mode; since the previous snapshot lacks the property, // the max sequence number is recomputed by scanning the base manifests. - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::WRITE_SEQUENCE_NUMBER_INIT_MODE] = "snapshot"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder + + .Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4926,12 +5243,15 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { } TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4952,12 +5272,15 @@ TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { } TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "false") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + context_builder_options[Options::OVERWRITE_UPGRADE] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4985,12 +5308,15 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - CommitContextBuilder builder(pk_table_path, "test_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - builder.AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::OVERWRITE_UPGRADE, "true") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions builder_load_options{pk_table_path}; + std::map builder_options; + CommitContextBuilder builder("test_user"); + builder_options[Options::FILE_SYSTEM] = "local"; + builder_options[Options::OVERWRITE_UPGRADE] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, + FileStoreCommit::Create(builder_load_options, std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5005,12 +5331,15 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { } TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5028,10 +5357,14 @@ TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelt } TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5041,10 +5374,14 @@ TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests } TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path_}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index a7f39628d..e034167f7 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -29,6 +29,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/utils/linked_hash_map.h" #include "paimon/common/utils/path_util.h" @@ -80,12 +81,15 @@ TEST(FileStoreCommitTest, TestCreate) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - CommitContextBuilder context_builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); } @@ -95,14 +99,12 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { ASSERT_TRUE(dir); std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir->Str(), options)); - auto commit_context = std::make_unique( - dir->Str(), "commit_user", /*ignore_empty_commit=*/true, - /*use_rest_catalog_commit=*/false, catalog, /*identifier=*/std::nullopt, - /*table_id=*/std::nullopt, /*append_commit_check_conflict=*/false, GetDefaultPool(), - CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options, - /*format_table=*/nullptr); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), - "a catalog commit requires a table identifier"); + TableLoadOptions load_options{dir->Str()}; + load_options.catalog = catalog; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + CommitContextBuilder("commit_user").Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(load_options, std::move(context), options), + "a catalog load requires a table identifier"); } TEST(FileStoreCommitTest, TestCatalogCommitUsesPerTableFileSystem) { @@ -123,10 +125,14 @@ TEST(FileStoreCommitTest, TestCatalogCommitUsesPerTableFileSystem) { catalog->SetFileSystem(std::make_shared()); catalog->SetTableFileSystem(dir->GetFileSystem()); - CommitContextBuilder builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, - builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + CommitContextBuilder builder("commit_user"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(builder_load_options, std::move(ctx), + builder_options)); // The file system was resolved per table, not from the catalog-wide one. const std::vector& requests = catalog->TableFileSystemRequests(); @@ -161,12 +167,15 @@ TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - CommitContextBuilder context_builder(table_path, "commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + CommitContextBuilder context_builder("commit_user"); + context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(context_builder_load_options, + std::move(commit_context), context_builder_options)); LinkedHashMap dv_ranges; dv_ranges.insert_or_assign("data-file-1", diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 01f46b053..8ba2b0ab5 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -32,6 +32,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" @@ -88,14 +89,16 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { }; auto create_writer = [&](const std::shared_ptr& realtime_context) -> Result> { - WriteContextBuilder builder(table_path, "writer"); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr ctx, - builder.WithCatalog(catalog, identifier) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context) - .AddOption(Options::FILE_SYSTEM, "unregistered-file-system") - .Finish()); - return FileStoreWrite::Create(std::move(ctx)); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("writer"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + builder_options[Options::FILE_SYSTEM] = "unregistered-file-system"; + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr ctx, + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).Finish()); + return FileStoreWrite::Create(builder_load_options, std::move(ctx), builder_options); }; ASSERT_OK_AND_ASSIGN(auto realtime_context, RealtimeContext::Create()); ASSERT_OK_AND_ASSIGN(auto writer, create_writer(realtime_context)); @@ -106,10 +109,15 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { ASSERT_EQ(base_snapshot_id, std::nullopt); ASSERT_OK_AND_ASSIGN(auto progress, writer->PrepareCommitWithProgress(1)); ASSERT_EQ(progress.size(), 1u); - CommitContextBuilder commit_builder(table_path, "writer"); - ASSERT_OK_AND_ASSIGN(auto commit_ctx, - commit_builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_ctx))); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("writer"); + commit_builder_load_options.catalog = catalog; + commit_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(auto commit_ctx, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto commit, FileStoreCommit::Create(commit_builder_load_options, std::move(commit_ctx), + commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, commit->CommitWithProgress(progress, 1, std::nullopt)); ASSERT_EQ(snapshot_id, 1); @@ -190,13 +198,16 @@ TEST(FileStoreWriteTest, TestBucketBaseSnapshotTracksActualLazyRestore) { catalog->SetFileSystem(dir->GetFileSystem()); catalog->CheckBaseSnapshotUuid(); auto create_writer = [&](bool ignore_previous) -> Result> { - WriteContextBuilder builder(dir->Str(), "writer"); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.WithCatalog(catalog, identifier) - .WithStreamingMode(true) - .WithIgnorePreviousFiles(ignore_previous) - .Finish()); - return FileStoreWrite::Create(std::move(context)); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + WriteContextBuilder builder("writer"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr context, + builder.WithStreamingMode(true).WithIgnorePreviousFiles(ignore_previous).Finish()); + return FileStoreWrite::Create(builder_load_options, std::move(context), + builder_options); }; auto make_batch = [&](const std::string& dt, int32_t bucket, const std::string& json) -> Result> { @@ -210,10 +221,15 @@ TEST(FileStoreWriteTest, TestBucketBaseSnapshotTracksActualLazyRestore) { .SetBucket(bucket) .Finish(); }; - CommitContextBuilder commit_builder(dir->Str(), "writer"); - ASSERT_OK_AND_ASSIGN(auto commit_context, - commit_builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("writer"); + commit_builder_load_options.catalog = catalog; + commit_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_builder_load_options, + std::move(commit_context), + commit_builder_options)); auto seed_snapshot = [&](int32_t bucket, int64_t identifier) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, create_writer(false)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -275,14 +291,16 @@ TEST(FileStoreWriteTest, TestBucketBaseSnapshotTracksActualLazyRestore) { TEST(FileStoreWriteTest, TestCreateWithInvalidInput) { auto dir = UniqueTestDirectory::Create(); - WriteContextBuilder builder(dir->Str(), "commit_user_1"); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + WriteContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx1, builder.WithMemoryPool(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(std::move(ctx1))); + ASSERT_NOK(FileStoreWrite::Create(builder_load_options, std::move(ctx1), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx2, builder.WithExecutor(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(std::move(ctx2))); - ASSERT_NOK(FileStoreWrite::Create(/*context=*/nullptr)); + ASSERT_NOK(FileStoreWrite::Create(builder_load_options, std::move(ctx2), builder_options)); + ASSERT_NOK(FileStoreWrite::Create({}, /*context=*/nullptr)); } TEST(FileStoreWriteTest, TestCreateAppendTable) { @@ -303,14 +321,18 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { ASSERT_FALSE(shared_catalog->SupportsVersionManagement()); for (bool use_catalog : {false, true}) { SCOPED_TRACE(use_catalog); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); if (use_catalog) { - context_builder.WithCatalog(shared_catalog, Identifier("foo", "bar")); + context_builder_load_options.catalog = shared_catalog; + context_builder_load_options.identifier.emplace(Identifier("foo", "bar")); } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); ASSERT_OK(file_store_write->Close()); } } @@ -327,12 +349,15 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem catalog->SetTableSchema(schema); catalog->SetFileSystem(dir->GetFileSystem()); - WriteContextBuilder main_builder(dir->Str(), "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr main_context, - main_builder.WithCatalog(catalog, Identifier("db", "table$branch_main")).Finish()); + TableLoadOptions main_builder_load_options{dir->Str()}; + std::map main_builder_options; + WriteContextBuilder main_builder("commit_user_1"); + main_builder_load_options.catalog = catalog; + main_builder_load_options.identifier.emplace(Identifier("db", "table$branch_main")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_write, - FileStoreWrite::Create(std::move(main_context))); + FileStoreWrite::Create(main_builder_load_options, std::move(main_context), + main_builder_options)); ASSERT_OK(main_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().front(), Identifier("db", "table")); @@ -341,12 +366,15 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem ASSERT_OK(branch_schema_manager.CreateTable(logical_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); catalog->SetTableSchema(nullptr); - WriteContextBuilder branch_builder(dir->Str(), "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr branch_context, - branch_builder.WithCatalog(catalog, Identifier("db", "table$branch_dev")).Finish()); + TableLoadOptions branch_builder_load_options{dir->Str()}; + std::map branch_builder_options; + WriteContextBuilder branch_builder("commit_user_1"); + branch_builder_load_options.catalog = catalog; + branch_builder_load_options.identifier.emplace(Identifier("db", "table$branch_dev")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_write, - FileStoreWrite::Create(std::move(branch_context))); + FileStoreWrite::Create(branch_builder_load_options, + std::move(branch_context), branch_builder_options)); ASSERT_OK(branch_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); } @@ -369,11 +397,15 @@ TEST(FileStoreWriteTest, TestCatalogWriteUsesPerTableFileSystem) { catalog->SetFileSystem(std::make_shared()); catalog->SetTableFileSystem(dir->GetFileSystem()); - WriteContextBuilder builder(table_path, "writer"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, - builder.WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("writer"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK(writer->Close()); // The file system was resolved per table, not from the catalog-wide one. @@ -397,11 +429,13 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidBucket) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "-2").Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::BUCKET] = "-2"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options), "not support bucket -2 in append table"); } @@ -423,13 +457,14 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidWriteType) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::BUCKET] = "-1"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "-1") - .WithWriteSchema({"field_non_exist"}) - .Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + context_builder.WithWriteSchema({"field_non_exist"}).Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options), "write field field_non_exist does not exist in table schema"); } @@ -449,12 +484,14 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0"}, /*primary_keys=*/{"f0", "f1", "f4"}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::BUCKET, "2").Finish()); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options[Options::BUCKET] = "2"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); auto fs = std::make_shared(); SchemaManager schema_manager(fs, PathUtil::JoinPath(dir->Str(), "foo.db/bar")); @@ -483,10 +520,12 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTableWithInvalidBucket) { /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{"f1", "f4"}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user_1"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options), "not support bucket -1 in key value table"); } diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index fa86967f5..8d9873dde 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -78,7 +78,7 @@ std::string IdentifierToFlatJson(const Identifier& identifier) { } // namespace std::shared_ptr InternalReadContext::CreateDependencyCatalogContext() const { - const auto& catalog = read_context_->GetCatalog(); + const auto& catalog = catalog_; if (!catalog) { // Preserve path-only filesystem reads using /.db/
. return std::make_shared( @@ -88,13 +88,13 @@ std::shared_ptr InternalReadContext::CreateDependencyCatalogCont // Preserve the dependency catalog's effective settings: table options, then catalog // options, then explicit read overrides. The upstream reader loads its own table options; - // ReadContextBuilder must not copy these catalog settings back into that read. + // Catalog settings are used only while loading dependencies. auto dependency_options = options_.ToMap(); const auto& catalog_options = catalog->GetOptions(); for (const auto& [key, value] : catalog_options) { dependency_options[key] = value; } - const auto& read_options = read_context_->GetOptions(); + const auto& read_options = dynamic_options_; for (const auto& [key, value] : read_options) { dependency_options[key] = value; } @@ -102,11 +102,11 @@ std::shared_ptr InternalReadContext::CreateDependencyCatalogCont std::string(RestApi::kHeaderOptionPrefix) + RestApi::kReadViaHeader; auto metastore = dependency_options.find(CatalogOptions::METASTORE); if (metastore != dependency_options.end() && - StringUtils::ToLowerCase(metastore->second) == "rest" && read_context_->GetIdentifier() && + StringUtils::ToLowerCase(metastore->second) == "rest" && identifier_ && dependency_options.count(read_via_option) == 0) { // Only fill in a missing header when the merged options select a REST dependency catalog. dependency_options[read_via_option] = - UrlUtils::EncodeString(IdentifierToFlatJson(*read_context_->GetIdentifier())); + UrlUtils::EncodeString(IdentifierToFlatJson(*identifier_)); } // A REST warehouse identifies the catalog, independent of the table's storage location. return std::make_shared(catalog->GetRootPath(), dependency_options, @@ -277,11 +277,15 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, - const std::map& options) { + const std::map& options, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, const std::optional& identifier, + const std::map& dynamic_options) { PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - return CreateFromCoreOptions(context, table_schema, std::move(core_options)); + CoreOptions::FromMap(options, file_system, fs_scheme_map)); + return CreateForTable(context, table_schema, std::move(core_options), root_path, fs_scheme_map, + catalog, identifier, dynamic_options); } Result> InternalReadContext::CreateFromCoreOptions( @@ -490,8 +494,7 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea table_schema_(table_schema), read_schema_(read_schema), arrow_pool_(arrow_pool), - options_(options), - fs_scheme_map_(read_context->GetFileSystemSchemeToIdentifierMap()) {} + options_(options) {} Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, @@ -507,62 +510,71 @@ Result> InternalReadContext::CreateWithSche original->options_, original->arrow_pool_)); result->path_ = original->path_; result->fs_scheme_map_ = original->fs_scheme_map_; + result->catalog_ = original->catalog_; + if (original->identifier_) { + result->identifier_.emplace(*original->identifier_); + } + result->dynamic_options_ = original->dynamic_options_; return result; } Result> InternalReadContext::CreateForTable( const std::shared_ptr& context, const std::shared_ptr& schema, CoreOptions options, const std::string& root_path, - const std::map& fs_scheme_map) { + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, const std::optional& identifier, + const std::map& dynamic_options) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr result, CreateFromCoreOptions(context, schema, std::move(options))); result->path_ = root_path; result->fs_scheme_map_ = fs_scheme_map; + result->catalog_ = catalog; + if (identifier) { + result->identifier_.emplace(*identifier); + } + result->dynamic_options_ = dynamic_options; return result; } Result> InternalReadContext::CreateForFallbackBranch( const std::shared_ptr& context, const std::string& branch, const std::shared_ptr& file_system, const std::string& root_path, - const std::map& fs_scheme_map, bool use_context_load) { + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, const std::optional& identifier, + const std::map& dynamic_options, + const std::optional& supplied_main_schema, + const std::optional>& fallback_file_system) { if (context->GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); } - std::shared_ptr fallback_fs = file_system; - if (use_context_load) { - // Preserve static-factory loading: metadata uses the caller's storage configuration; - // data storage is resolved below after the fallback schema options are known. + std::shared_ptr metadata_fs = file_system; + if (fallback_file_system) { PAIMON_ASSIGN_OR_RAISE( CoreOptions metadata_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - fs_scheme_map)); - fallback_fs = metadata_options.GetFileSystem(); - } - std::shared_ptr schema; - if (use_context_load && branch == BranchManager::DEFAULT_MAIN_BRANCH && - context->GetSpecificTableSchema()) { - PAIMON_ASSIGN_OR_RAISE( - schema, TableSchema::CreateFromJson(context->GetSpecificTableSchema().value())); + CoreOptions::FromMap(dynamic_options, *fallback_file_system, fs_scheme_map)); + metadata_fs = metadata_options.GetFileSystem(); + } + SchemaManager manager(metadata_fs, root_path, branch); + std::optional> schema; + if (BranchManager::IsMainBranch(branch) && supplied_main_schema) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr supplied, + TableSchema::CreateFromJson(*supplied_main_schema)); + schema = std::shared_ptr(std::move(supplied)); + } else { + PAIMON_ASSIGN_OR_RAISE(schema, manager.Latest()); } if (!schema) { - SchemaManager manager(fallback_fs, root_path, branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - manager.Latest()); - if (!latest) { - return Status::Invalid("fallback schema not found"); - } - schema = latest.value(); + return Status::Invalid("fallback schema not found"); } - auto options = schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { + auto options = schema.value()->Options(); + for (const auto& [key, value] : dynamic_options) { options[key] = value; } options[Options::BRANCH] = branch; PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap(options, - use_context_load ? context->GetSpecificFileSystem() : file_system, - fs_scheme_map)); - return CreateForTable(context, schema, std::move(core_options), root_path, fs_scheme_map); + CoreOptions::FromMap(options, fallback_file_system.value_or(file_system), fs_scheme_map)); + return CreateForTable(context, schema.value(), std::move(core_options), root_path, + fs_scheme_map, catalog, identifier, dynamic_options); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 5e6925f6f..7bb581705 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -24,6 +24,7 @@ #include #include +#include "paimon/catalog/identifier.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" #include "paimon/read_context.h" @@ -35,6 +36,7 @@ class Schema; } // namespace arrow namespace paimon { +class Catalog; struct CatalogContext; class Executor; class MemoryPool; @@ -59,16 +61,29 @@ class InternalReadContext { static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options); + const std::map& options, + const std::shared_ptr& file_system = nullptr, const std::string& root_path = "", + const std::map& fs_scheme_map = {}, + const std::shared_ptr& catalog = nullptr, + const std::optional& identifier = std::nullopt, + const std::map& dynamic_options = {}); static Result> CreateForTable( const std::shared_ptr& context, const std::shared_ptr& schema, CoreOptions options, const std::string& root_path, - const std::map& fs_scheme_map); + const std::map& fs_scheme_map, + const std::shared_ptr& catalog = nullptr, + const std::optional& identifier = std::nullopt, + const std::map& dynamic_options = {}); static Result> CreateForFallbackBranch( const std::shared_ptr& context, const std::string& branch, const std::shared_ptr& file_system, const std::string& root_path, - const std::map& fs_scheme_map, bool use_context_load); + const std::map& fs_scheme_map, + const std::shared_ptr& catalog = nullptr, + const std::optional& identifier = std::nullopt, + const std::map& dynamic_options = {}, + const std::optional& supplied_main_schema = std::nullopt, + const std::optional>& fallback_file_system = std::nullopt); /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { @@ -88,7 +103,7 @@ class InternalReadContext { return table_schema_; } const std::string& GetPath() const { - return path_ ? *path_ : read_context_->GetPath(); + return path_; } std::shared_ptr CreateDependencyCatalogContext() const; @@ -197,7 +212,10 @@ class InternalReadContext { std::shared_ptr read_schema_; std::shared_ptr arrow_pool_; CoreOptions options_; - std::optional path_; + std::string path_; + std::shared_ptr catalog_; + std::optional identifier_; + std::map dynamic_options_; std::map fs_scheme_map_; std::map index_score_bindings_; }; diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index 2147e8242..b393ae69e 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -29,6 +29,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/catalog_options.h" #include "paimon/common/catalog/catalog_context.h" #include "paimon/common/table/special_fields.h" @@ -52,9 +53,11 @@ namespace paimon::test { TEST(InternalReadContext, TestReadWithUnspecifiedSchema) { // no read schema is specified, read all fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -74,13 +77,18 @@ TEST(InternalReadContext, TestPreciseScalarEvaluation) { arrow::field("f1", arrow::utf8())}), {}, {}, {})); for (bool enabled : {false, true}) { - ReadContextBuilder builder("unused-table-path"); + TableLoadOptions builder_load_options{"unused-table-path"}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(enabled).SetPredicate( PredicateBuilder::IsNotNull(0, "f0", FieldType::INT)); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr internal_context, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create( + context, table_schema, table_schema->Options(), builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); ASSERT_EQ(internal_context->EnablePreciseFileIndexEvaluate(), enabled); ASSERT_FALSE(internal_context->EnablePredicateFilter()); ASSERT_TRUE(internal_context->GetCoreOptions().FileIndexReadEnabled()); @@ -92,33 +100,46 @@ TEST(InternalReadContext, TestPreciseScalarEvaluation) { } auto match = std::make_shared("f1", R"({"match":{"query":"apple"}})"); ASSERT_OK_AND_ASSIGN(auto full_text, PredicateBuilder::FullTextMatch(1, match)); - ReadContextBuilder builder("unused-table-path"); + TableLoadOptions builder_load_options{"unused-table-path"}; + std::map builder_options; + ReadContextBuilder builder; builder.SetPredicate(full_text); ASSERT_NOK_WITH_MSG(builder.Finish(), "EnablePreciseFileIndexEvaluate(true)"); builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_TRUE(context->EnablePreciseFileIndexEvaluate()); - ASSERT_OK(InternalReadContext::Create(context, table_schema, table_schema->Options())); + ASSERT_OK(InternalReadContext::Create( + context, table_schema, table_schema->Options(), builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); } TEST(InternalReadContext, TestPreciseEvaluationRequiresFileIndexRead) { ASSERT_OK_AND_ASSIGN( std::shared_ptr table_schema, TableSchema::Create(0, arrow::schema({arrow::field("f0", arrow::int32())}), {}, {}, {})); - ReadContextBuilder builder("unused-table-path"); + TableLoadOptions builder_load_options{"unused-table-path"}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, - {{Options::FILE_INDEX_READ_ENABLED, "false"}}), - "requires file-index.read.enabled=true"); + ASSERT_NOK_WITH_MSG( + InternalReadContext::Create( + context, table_schema, {{Options::FILE_INDEX_READ_ENABLED, "false"}}, + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options), + "requires file-index.read.enabled=true"); } TEST(InternalReadContext, TestReadWithSpecifiedSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -146,10 +167,13 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { {CatalogOptions::URI, "http://read"}, {CatalogOptions::TOKEN, "read-token"}, {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}}; - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1")).SetOptions(read_options).Finish()); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(Identifier("db1", "t1")); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); std::map options = table_schema->Options(); options[Options::READ_BATCH_SIZE] = "64"; options[CatalogOptions::DATA_TOKEN_ENABLED] = "false"; @@ -157,8 +181,12 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { for (const auto& [key, value] : read_options) { options[key] = value; } - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); auto expected_options = options; @@ -166,7 +194,7 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { ASSERT_EQ(dependency_context->root_path, catalog->GetRootPath()); ASSERT_EQ(dependency_context->options, expected_options); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), "table-view"); - ASSERT_EQ(read_context->GetOptions(), read_options); + ASSERT_EQ(builder_options, read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -195,20 +223,26 @@ TEST(InternalReadContext, TestDependencyReadKeepsTheOutermostView) { if (test_case.read_header) { read_options["header.X-Paimon-Read-Via"] = test_case.read_header; } - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "nested-view")) - .SetOptions(read_options) - .Finish()); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(Identifier("db1", "nested-view")); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); auto options = table_schema->Options(); options["header.X-Paimon-Read-Via"] = test_case.read_header ? test_case.read_header : "table-view"; - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), test_case.expected_header); - ASSERT_EQ(read_context->GetOptions(), read_options); + ASSERT_EQ(builder_options, read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -252,13 +286,19 @@ TEST(InternalReadContext, TestDependencyReadHeaderUsesFinalMetastore) { read_options[CatalogOptions::METASTORE] = test_case.read_metastore; options[CatalogOptions::METASTORE] = test_case.read_metastore; } - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1")) - .SetOptions(read_options) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(read_context, table_schema, options)); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(Identifier("db1", "t1")); + builder_options = read_options; + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create( + read_context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); if (test_case.expected_metastore) { ASSERT_EQ(dependency_context->options.at(CatalogOptions::METASTORE), @@ -271,7 +311,7 @@ TEST(InternalReadContext, TestDependencyReadHeaderUsesFinalMetastore) { } else { ASSERT_EQ(dependency_context->options.count("header.X-Paimon-Read-Via"), 0u); } - ASSERT_EQ(read_context->GetOptions(), read_options); + ASSERT_EQ(builder_options, read_options); ASSERT_EQ(catalog->GetOptions(), catalog_options); ASSERT_EQ(internal_context->GetCoreOptions().ToMap(), options); } @@ -281,10 +321,10 @@ TEST(InternalReadContext, TestDependencyReadHeaderNeedsAnIdentifier) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(auto read_context, - builder.AddOption(CatalogOptions::METASTORE, "REST").Finish()); - ASSERT_FALSE(read_context->GetIdentifier()); + TableLoadOptions builder_load_options{path}; + ReadContextBuilder builder; + ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); + ASSERT_FALSE(builder_load_options.identifier); auto options = table_schema->Options(); options[CatalogOptions::METASTORE] = "REST"; ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), @@ -300,12 +340,12 @@ TEST(InternalReadContext, TestDependencyReadHeaderReportsTheBranchOfTheStartingT auto catalog = std::make_shared(); catalog->SetTableFileSystem(std::make_shared()); catalog->SetOptions({{CatalogOptions::METASTORE, "REST"}}); - ReadContextBuilder builder(path); - ASSERT_OK_AND_ASSIGN(auto read_context, - builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + ReadContextBuilder builder; + ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options(), + nullptr, path, {}, catalog, Identifier("db1", "t1", "dev"))); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), "%7B%22database%22%3A%22db1%22%2C%22object%22%3A%22t1%24branch_dev%22%7D"); @@ -319,14 +359,18 @@ TEST(InternalReadContext, TestFileSystemSchemeMapReachesTheDependencyRead) { // surfaces as a `dfs://` dependency table that cannot be opened, far away from the cause. std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; const std::map scheme_map = {{"file", "local"}}; - ReadContextBuilder context_builder(path); - context_builder.WithFileSystemSchemeToIdentifierMap(scheme_map); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; + context_builder_load_options.fs_scheme_to_identifier_map = scheme_map; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options(), + nullptr, path, scheme_map)); ASSERT_EQ(internal_context->GetFileSystemSchemeToIdentifierMap(), scheme_map); ASSERT_EQ(internal_context->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map, scheme_map); @@ -334,10 +378,12 @@ TEST(InternalReadContext, TestFileSystemSchemeMapReachesTheDependencyRead) { TEST(InternalReadContext, TestReadWithSpecifiedFieldId) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldIds({3, 0}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -350,13 +396,15 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldId) { TEST(InternalReadContext, TestReadWithSpecifiedFieldIdAndSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; // read schema is specified, read fields in schema // will use field ids instead of field names. context_builder.SetReadFieldNames({"f0"}); context_builder.SetReadFieldIds({3, 0}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -371,11 +419,13 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { { // test simple std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames( {"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER", "_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); auto new_options = table_schema->Options(); new_options[Options::ROW_TRACKING_ENABLED] = "true"; @@ -393,10 +443,12 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { { // test invalid case: disable row tracking while read row tracking fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options()), @@ -405,10 +457,12 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { { // test invalid case: disable data evolution while read score fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options()), @@ -423,7 +477,9 @@ TEST(InternalReadContext, TestPreciseEvaluationRejectsUnnamedScoreProjection) { for (const auto& search_type : std::vector{"none", "scalar", "vector", "full-text", "full-text-match"}) { for (const auto& projection : std::vector{"names", "ids", "schema"}) { - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f0"}); if (projection == "names") { builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}); @@ -457,7 +513,11 @@ TEST(InternalReadContext, TestPreciseEvaluationRejectsUnnamedScoreProjection) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(context, table_schema, table_schema->Options()), + InternalReadContext::Create( + context, table_schema, table_schema->Options(), + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options), "not exist in table schema"); } } @@ -471,21 +531,30 @@ TEST(InternalReadContext, TestPrimaryKeyVectorSearchRequiresDeletionVectors) { std::shared_ptr search = std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"f0", "_f3_INDEX_SCORE"}) .EnablePreciseFileIndexEvaluate(true) .SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create( + context, table_schema, table_schema->Options(), builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options)); EXPECT_TRUE(internal->HasVectorOrFullTextSearch()); EXPECT_TRUE(internal->EnablePreciseFileIndexEvaluate()); std::map options = table_schema->Options(); options[Options::DELETION_VECTORS_ENABLED] = "false"; - ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, options), - "primary-key tables requires deletion vectors"); + ASSERT_NOK_WITH_MSG( + InternalReadContext::Create( + context, table_schema, options, builder_load_options.file_system, + builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, + builder_load_options.catalog, builder_load_options.identifier, builder_options), + "primary-key tables requires deletion vectors"); } TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { @@ -511,7 +580,9 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { std::make_shared(test_case.field_name, R"({"match":{"query":"apple"}})"); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch(test_case.field_index, match)); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames(test_case.projection) .SetPredicate(predicate); @@ -519,7 +590,11 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { if (test_case.expected_error.empty()) { ASSERT_OK_AND_ASSIGN( std::shared_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create( + context, table_schema, table_schema->Options(), + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options)); ASSERT_TRUE(internal->GetIndexScoreBindings().empty()); ASSERT_FALSE(internal->NeedsScores( {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH, match->field_name})); @@ -528,7 +603,11 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { "CreateWithSchema does not support"); } else { ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(context, table_schema, table_schema->Options()), + InternalReadContext::Create( + context, table_schema, table_schema->Options(), + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options), test_case.expected_error); } } @@ -542,7 +621,9 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { for (bool vector_search : {false, true}) { for (const auto& projection_type : std::vector{"none", "names", "schema"}) { const bool project_score = projection_type != "none"; - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; std::vector projection = {"f1"}; if (project_score) { projection.push_back(vector_search ? "_f3_INDEX_SCORE" : "_f0_INDEX_SCORE"); @@ -573,7 +654,11 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr internal, - InternalReadContext::Create(context, table_schema, table_schema->Options())); + InternalReadContext::Create( + context, table_schema, table_schema->Options(), + builder_load_options.file_system, builder_load_options.path, + builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, + builder_load_options.identifier, builder_options)); ASSERT_EQ(internal->GetReadSchema()->field_names(), projection); ASSERT_TRUE(internal->NeedsScores(source)); ASSERT_FALSE(internal->NeedsScores( @@ -634,7 +719,7 @@ TEST_F(InternalReadContextScoreTest, BindsFullTextScoresByProjection) { {2, false, true}, {2, true, false}, {2, true, true}}; for (const auto& test_case : cases) { const int32_t score_count = test_case.score_count; - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; std::vector projection = {"F1", "F2"}; if (score_count >= 1) { projection.push_back(test_case.reverse_scores ? "_F1_INDEX_SCORE" : "_F2_INDEX_SCORE"); @@ -696,7 +781,7 @@ TEST_F(InternalReadContextScoreTest, BindsRootAndPredicateScores) { ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::FullTextMatch(0, match)); auto search = std::make_shared("F2", match->query, /*limit=*/10); - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.SetPredicate(predicate) .EnablePreciseFileIndexEvaluate(true) .SetFullTextSearch(search) @@ -756,7 +841,7 @@ TEST_F(InternalReadContextScoreTest, RejectsAmbiguousOrConditionalScoreProjectio {valid, false, ""}}; for (const auto& test_case : cases) { for (bool project_score : {false, true}) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetPredicate(test_case.predicate) .SetReadFieldNames(project_score ? std::vector{"F1", "_F1_INDEX_SCORE"} @@ -790,7 +875,7 @@ TEST_F(InternalReadContextScoreTest, RejectsPredicatesOnFullTextMatchScores) { for (const auto& score_predicate : score_predicates) { ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({text, score_predicate})); - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"F1", "_F1_INDEX_SCORE"}) .SetPredicate(predicate); @@ -808,7 +893,7 @@ TEST_F(InternalReadContextScoreTest, RejectsNestedPredicatesOnRootSearchScores) ASSERT_OK_AND_ASSIGN(std::shared_ptr child, PredicateBuilder::Or({scalar, score})); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({scalar, child})); - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetFullTextSearch(search) .SetReadFieldNames({"F1", "_F1_INDEX_SCORE"}) @@ -819,7 +904,7 @@ TEST_F(InternalReadContextScoreTest, RejectsNestedPredicatesOnRootSearchScores) } TEST_F(InternalReadContextScoreTest, RejectsPredicatesOnUnnamedIndexScores) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.SetPredicate(PredicateBuilder::IsNull(0, "_INDEX_SCORE", FieldType::FLOAT)); ASSERT_NOK_WITH_MSG( Create(&builder), @@ -828,7 +913,7 @@ TEST_F(InternalReadContextScoreTest, RejectsPredicatesOnUnnamedIndexScores) { TEST_F(InternalReadContextScoreTest, RejectsMissingScoreSource) { for (const auto& name : {"_F1_INDEX_SCORE", "_missing_INDEX_SCORE"}) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.SetReadFieldNames({name}); ASSERT_NOK_WITH_MSG(Create(&builder), fmt::format("Index score field '{}' has no search source", name)); @@ -840,7 +925,7 @@ TEST_F(InternalReadContextScoreTest, ResolvesScoresOnlyFromEffectiveProjection) InternalReadContext::IndexScoreSource::Type::FULL_TEXT_SEARCH, "F2"}; auto search = std::make_shared("F2", R"({"match":{"query":"apple"}})", 1); for (bool explicit_schema : {false, true}) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetFullTextSearch(search) .SetReadFieldNames({"_INDEX_SCORE", "_missing_INDEX_SCORE"}) @@ -870,7 +955,7 @@ TEST_F(InternalReadContextScoreTest, ResolvesScoresOnlyFromEffectiveProjection) } TEST_F(InternalReadContextScoreTest, SchemaReplacementUsesProvidedSchema) { - ReadContextBuilder builder("/tmp/unused-table-path"); + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr internal, Create(&builder)); for (const auto& schema : {arrow::schema({arrow::field("F1", arrow::utf8())}), @@ -884,10 +969,12 @@ TEST_F(InternalReadContextScoreTest, SchemaReplacementUsesProvidedSchema) { TEST(InternalReadContext, TestReadWithValueKindField) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "_VALUE_KIND", "f0"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -903,14 +990,16 @@ TEST(InternalReadContext, TestReadWithFieldIdsAndSpecialFields) { { // test simple std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + ReadContextBuilder context_builder; // here we use field ids instead of field names, and specify special ids for row id, // sequence number and index score. context_builder.SetReadFieldIds({3, 0, SpecialFieldIds::ROW_ID, SpecialFieldIds::SEQUENCE_NUMBER, SpecialFieldIds::INDEX_SCORE}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); auto new_options = table_schema->Options(); new_options[Options::ROW_TRACKING_ENABLED] = "true"; @@ -941,12 +1030,19 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(read_context, table_schema, table_schema->Options()), + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options), "not exist in table schema"); } @@ -959,13 +1055,20 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, enabled_options)); + InternalReadContext::Create( + read_context, table_schema, enabled_options, + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options)); auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); ASSERT_TRUE(internal_context->GetReadSchema()->Equals(expected_schema)); } @@ -979,17 +1082,25 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaWithoutFieldIds) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options)); std::vector expected_fields = { DataField(3, arrow::field("f3", arrow::float64())), @@ -1011,17 +1122,25 @@ TEST(InternalReadContext, TestProjectedSchemaMetadataWhitelist) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + context_builder_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options)); auto aligned_field = internal_context->GetReadSchema()->GetFieldByName("f0"); ASSERT_TRUE(aligned_field); @@ -1056,13 +1175,20 @@ TEST(InternalReadContext, TestMapSharedShreddingAccessRequiresSharedShreddingLay auto c_read_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*arrow::schema({access_field}), c_read_schema.get()).ok()); - ReadContextBuilder context_builder("/tmp/unused-table-path"); + TableLoadOptions context_builder_load_options{"/tmp/unused-table-path"}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_read_schema)); ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); std::shared_ptr read_context = std::move(unique_read_context); ASSERT_NOK_WITH_MSG( - InternalReadContext::Create(read_context, table_schema, table_schema->Options()), + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + context_builder_load_options.file_system, context_builder_load_options.path, + context_builder_load_options.fs_scheme_to_identifier_map, + context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_options), "Selected-key MAP pushdown only supports top-level shared-shredding MAP field: tags"); } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index 7602a2763..bb863ae12 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -41,6 +41,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/data/shredding/map_shredding_defs.h" @@ -135,14 +136,17 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { /*primary_keys=*/{"f0"}, table_options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); if (with_temp_directory) { context_builder.WithTempDirectory(dir->Str()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(std::move(write_context)); + return FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options); } Status WriteSingleStringRow(FileStoreWrite* file_store_write, int32_t bucket, @@ -196,11 +200,15 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { - WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("test"); + builder_options = options; + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( /*wait_compaction=*/false, commit_identifier)); @@ -219,11 +227,15 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { - CommitContextBuilder builder(table_path, "test"); - builder.SetOptions(options); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + CommitContextBuilder builder("test"); + builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(builder_load_options, + std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -340,10 +352,13 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{"f1"}, /*options=*/{{"bucket", "1"}}, /*ignore_if_exists=*/false)); - WriteContextBuilder builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), commit_user); + TableLoadOptions builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map builder_options; + WriteContextBuilder builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, std::move(write_context), + builder_options)); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); } { @@ -358,11 +373,14 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{"f1"}, /*options=*/{{"bucket", "-2"}}, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - commit_user); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -390,11 +408,14 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{"f1"}, /*options=*/{{"bucket", "2"}}, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - commit_user); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); auto array = std::make_shared(); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { @@ -445,14 +466,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options) - .WithStreamingMode(true) + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("test"); + builder_options = options; + builder.WithStreamingMode(true) .WithRealtimeContext(realtime_context) .WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); std::unique_ptr batch = MakeBatch(realtime_schema, R"([ @@ -508,15 +532,19 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { SCOPED_TRACE(expected_error); ASSERT_OK_AND_ASSIGN(std::shared_ptr invalid_context, RealtimeContext::Create()); - WriteContextBuilder invalid_builder(table_path, "invalid"); - invalid_builder.SetOptions(options) - .WithStreamingMode(true) + TableLoadOptions invalid_builder_load_options{table_path}; + std::map invalid_builder_options; + WriteContextBuilder invalid_builder("invalid"); + invalid_builder_options = options; + invalid_builder.WithStreamingMode(true) .WithRealtimeContext(invalid_context) .WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_write_context, invalid_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_writer, - FileStoreWrite::Create(std::move(invalid_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr invalid_writer, + FileStoreWrite::Create(invalid_builder_load_options, std::move(invalid_write_context), + invalid_builder_options)); ASSERT_OK(invalid_writer->Write(MakeBatch(realtime_schema, R"([[30, 1, "existing"]])"))); ASSERT_OK_AND_ASSIGN(std::vector prepared, invalid_writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); @@ -547,14 +575,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path, "test"); - builder.SetOptions(options) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context) - .WithMemoryPool(pool); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("test"); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).WithMemoryPool(pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); const int64_t allocations_before_write = pool->allocation_count; ASSERT_OK(writer->Write(MakeBatch(realtime_schema, R"([[0, 1, "one"]])"))); @@ -570,15 +599,19 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr rejecting_pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr rejecting_context, RealtimeContext::Create()); - WriteContextBuilder rejecting_builder(table_path, "rejecting"); - rejecting_builder.SetOptions(options) - .WithStreamingMode(true) + TableLoadOptions rejecting_builder_load_options{table_path}; + std::map rejecting_builder_options; + WriteContextBuilder rejecting_builder("rejecting"); + rejecting_builder_options = options; + rejecting_builder.WithStreamingMode(true) .WithRealtimeContext(rejecting_context) .WithMemoryPool(rejecting_pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr rejecting_write_context, rejecting_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr rejecting_writer, - FileStoreWrite::Create(std::move(rejecting_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr rejecting_writer, + FileStoreWrite::Create(rejecting_builder_load_options, std::move(rejecting_write_context), + rejecting_builder_options)); ASSERT_OK(rejecting_writer->Write(MakeBatch(realtime_schema, "[]"))); const int64_t rejecting_allocations_before_write = rejecting_pool->allocation_count; rejecting_pool->reject_allocations = true; @@ -608,13 +641,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_context, RealtimeContext::Create()); - WriteContextBuilder initial_builder(table_path, "initial"); - initial_builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext( - initial_context); + TableLoadOptions initial_builder_load_options{table_path}; + std::map initial_builder_options; + WriteContextBuilder initial_builder("initial"); + initial_builder_options = options; + initial_builder.WithStreamingMode(true).WithRealtimeContext(initial_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_write_context, initial_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr initial_writer, - FileStoreWrite::Create(std::move(initial_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr initial_writer, + FileStoreWrite::Create(initial_builder_load_options, std::move(initial_write_context), + initial_builder_options)); ASSERT_OK(initial_writer->Write(MakeBatch(realtime_schema, R"([[0, 0, "initial"]])"))); ASSERT_OK_AND_ASSIGN(std::vector initial_progress, initial_writer->PrepareCommitWithProgress(0)); @@ -626,22 +663,31 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { initial_message->GetNewFilesIncrement().NewFiles()[0]->AssignSequenceNumber(max - 2, max - 2); initial_progress[0].offset_range = OffsetRange(0, max - 1); - CommitContextBuilder commit_builder(table_path, "initial"); - commit_builder.SetOptions(options); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("initial"); + commit_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr committer, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr committer, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, committer->CommitWithProgress(initial_progress, 0, std::nullopt)); ASSERT_OK(initial_writer->Close()); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path, "boundary"); - builder.SetOptions(options).WithStreamingMode(true).WithRealtimeContext(realtime_context); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder("boundary"); + builder_options = options; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); ASSERT_OK( writer->Write(MakeBatch(realtime_schema, fmt::format(R"([[{}, 1, "legal"]])", max - 1)))); ASSERT_OK_AND_ASSIGN(auto store_rows, ReadRealtimePrimaryKeyStoreRows(realtime_context)); @@ -750,11 +796,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillSimple) { {Options::WRITE_BUFFER_SPILLABLE, "true"}}, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); auto key_value_file_store_write = dynamic_cast(file_store_write.get()); auto get_writer = [&](int32_t bucket) -> std::shared_ptr { auto partition_iter = key_value_file_store_write->writers_.find(BinaryRow::EmptyRow()); @@ -809,11 +859,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - WriteContextBuilder first_context_builder(table_path, "first-writer"); - first_context_builder.SetOptions(options); + TableLoadOptions first_context_builder_load_options{table_path}; + std::map first_context_builder_options; + WriteContextBuilder first_context_builder("first-writer"); + first_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr first_context, first_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto first_write, FileStoreWrite::Create(std::move(first_context))); + ASSERT_OK_AND_ASSIGN( + auto first_write, + FileStoreWrite::Create(first_context_builder_load_options, std::move(first_context), + first_context_builder_options)); ASSERT_OK(WriteSingleStringRow(first_write.get(), /*bucket=*/0, "alice")); ASSERT_OK_AND_ASSIGN(auto commit_messages, first_write->PrepareCommit(/*wait_compaction=*/false, 1)); @@ -826,11 +882,17 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { SimpleStats::EmptyStats()); Commit(table_path, options, commit_messages); - WriteContextBuilder restored_context_builder(table_path, "restored-writer"); - restored_context_builder.SetOptions(options); + TableLoadOptions restored_context_builder_load_options{table_path}; + std::map restored_context_builder_options; + WriteContextBuilder restored_context_builder("restored-writer"); + restored_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr restored_context, restored_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto restored_write, FileStoreWrite::Create(std::move(restored_context))); + ASSERT_OK_AND_ASSIGN( + auto restored_write, + FileStoreWrite::Create(restored_context_builder_load_options, std::move(restored_context), + restored_context_builder_options)); auto key_value_write = dynamic_cast(restored_write.get()); ASSERT_NE(nullptr, key_value_write); ASSERT_OK_AND_ASSIGN(std::shared_ptr restore_files, @@ -859,11 +921,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillDiskQuotaExhaustedFallsBackToFlushDa {Options::WRITE_BUFFER_SPILL_MAX_DISK_SIZE, "1b"}}, /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); // Disk quota is 1 byte, so spill will exhaust quota immediately and fall back to // FlushWriteBuffer (writing data files directly instead of spill temp files). @@ -907,11 +973,15 @@ TEST_F(KeyValueFileStoreWriteTest, TestMultiRoundSpillWithSameKeyDeduplication) {Options::WRITE_BUFFER_SPILLABLE, "true"}}, /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "test"); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("test"); context_builder.WithTempDirectory(dir->Str()).WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); // Round 1: alice, bob, alice (duplicate key) → after dedup: alice + bob = 2 rows ASSERT_OK(WriteSingleStringRow(file_store_write.get(), /*bucket=*/0, "alice")); diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp index 1e34704cb..850eff66b 100644 --- a/src/paimon/core/operation/merge_file_split_read_test.cpp +++ b/src/paimon/core/operation/merge_file_split_read_test.cpp @@ -31,6 +31,7 @@ #include "arrow/c/abi.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/reader/concat_batch_reader.h" @@ -91,23 +92,31 @@ class MergeFileSplitReadTest : public ::testing::Test, } std::shared_ptr CreateInternalReadContext( - const std::shared_ptr& read_context, int32_t schema_id = 0) { - SchemaManager schema_manager(fs_, read_context->GetPath()); + const std::shared_ptr& read_context, + const TableLoadOptions& read_context_load_options, + const std::map& read_context_options, int32_t schema_id = 0) { + SchemaManager schema_manager(fs_, read_context_load_options.path); EXPECT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(schema_id)); - EXPECT_OK_AND_ASSIGN(auto context, InternalReadContext::Create(read_context, table_schema, - read_context->GetOptions())); + EXPECT_OK_AND_ASSIGN( + auto context, InternalReadContext::Create( + read_context, table_schema, read_context_options, + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); return context; } - void AddOptions(ReadContextBuilder* context_builder) const { + void AddOptions(ReadContextBuilder* context_builder, + std::map& options) const { auto [use_min_heap, enable_io_prefetch, enable_multi_thread_row_to_batch] = GetParam(); if (use_min_heap) { - context_builder->AddOption(Options::SORT_ENGINE, "min-heap"); + options[Options::SORT_ENGINE] = "min-heap"; } else { - context_builder->AddOption(Options::SORT_ENGINE, "loser-tree"); + options[Options::SORT_ENGINE] = "loser-tree"; } if (enable_io_prefetch) { - context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); + options["test.enable-adaptive-prefetch-strategy"] = "false"; context_builder->EnablePrefetch(true); context_builder->SetPrefetchBatchCount(/*batch_count=*/3); } else { @@ -609,7 +618,10 @@ TEST_F(MergeFileSplitReadTest, TestGenerateKeyPredicates2) { TEST_P(MergeFileSplitReadTest, TestSimple) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -620,13 +632,17 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN( std::unique_ptr split_read, @@ -675,7 +691,10 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), DataField(5, arrow::field("s1", arrow::utf8())), @@ -685,11 +704,15 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "s1", "v0"}); - context_builder.SetOptions( - {{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - std::shared_ptr internal_context = CreateInternalReadContext(read_context); + std::shared_ptr internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr split_read, CreateMergeFileSplitRead(internal_context)); @@ -742,7 +765,10 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) TEST_P(MergeFileSplitReadTest, TestLookUp) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -753,14 +779,18 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}, - {Options::FORCE_LOOKUP, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}, + {Options::FORCE_LOOKUP, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -790,15 +820,22 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}, - {Options::READ_BATCH_SIZE, "1"}}); - AddOptions(&context_builder); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}, + {Options::READ_BATCH_SIZE, "1"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); @@ -816,7 +853,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -827,11 +867,15 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "deduplicate"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::MERGE_ENGINE, "deduplicate"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -856,7 +900,10 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -868,11 +915,12 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + context_builder.EnableLateMaterializing(false); - AddOptions(&context_builder); + AddOptions(&context_builder, context_builder_options); // less_than will be ignore as it is partition predicate auto less_than = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"p1", @@ -888,9 +936,12 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate_result, PredicateBuilder::And({less_than, greater_or_equal, greater_than})); context_builder.SetPredicate(predicate_result); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -919,7 +970,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -931,10 +985,11 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); context_builder.EnableLateMaterializing(true); // key predicate, always pushed down into the data files auto greater_or_equal = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"k1", @@ -946,9 +1001,12 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { PredicateBuilder::And({greater_or_equal, greater_than})); context_builder.SetPredicate(predicate_result); context_builder.EnablePredicateFilter(true).EnableLateMaterializing(true); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -976,7 +1034,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k1", arrow::int32())), DataField(1, arrow::field("k0", arrow::int32())), @@ -991,14 +1052,18 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "k0", "p0", "p1", "s1", "s0", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, /*schema_id=*/7); + auto internal_context = CreateInternalReadContext(read_context, read_context_load_options, + read_context_options, /*schema_id=*/7); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1028,7 +1093,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(8, arrow::field("v2", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -1040,14 +1108,18 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"v2", "p1", "k0", "p0", "s0", "v0"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + AddOptions(&context_builder, context_builder_options); + + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, /*schema_id=*/7); + auto internal_context = CreateInternalReadContext(read_context, read_context_load_options, + read_context_options, /*schema_id=*/7); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1077,7 +1149,10 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -1088,14 +1163,18 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "aggregation"}, - {"fields.v1.aggregate-function", "bool_and"}, - {"fields.v0.aggregate-function", "sum"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "aggregation"}, + {"fields.v1.aggregate-function", "bool_and"}, + {"fields.v0.aggregate-function", "sum"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1125,7 +1204,10 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(1, arrow::field("k1", arrow::int32())), DataField(3, arrow::field("p1", arrow::int32())), @@ -1135,14 +1217,18 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "partial-update"}, - {"fields.v1.sequence-group", "v0"}, - {"fields.v0.aggregate-function", "first_value"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "partial-update"}, + {"fields.v1.sequence-group", "v0"}, + {"fields.v0.aggregate-function", "first_value"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1172,7 +1258,10 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -1183,11 +1272,15 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions( - {{Options::MERGE_ENGINE, "partial-update"}, {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1212,7 +1305,10 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -1223,11 +1319,15 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, - {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1252,7 +1352,10 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), DataField(1, arrow::field("k1", arrow::int32())), @@ -1263,11 +1366,15 @@ TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); - context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, - {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::MERGE_ENGINE, "partial-update"}, + {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); std::vector> empty_data_split; ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, empty_data_split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, @@ -1285,18 +1392,25 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { DataField(6, arrow::field("v0", arrow::float64())), DataField(7, arrow::field("v1", arrow::boolean()))}; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); - context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {Options::IGNORE_DELETE, "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {Options::IGNORE_DELETE, "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); auto fields_with_row_kind = read_schema->fields(); fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -1336,7 +1450,10 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { std::string path = paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(3, arrow::field("f3", arrow::float64())), DataField(2, arrow::field("f2", arrow::int32())), DataField(0, arrow::field("f0", arrow::utf8())), @@ -1344,10 +1461,13 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); context_builder.SetReadFieldNames({"f3", "f2", "f0", "f1"}); - context_builder.SetOptions({{Options::FILE_FORMAT, "orc"}, - {Options::MERGE_ENGINE, "deduplicate"}, - {"orc.read.enable-metrics", "true"}}); - AddOptions(&context_builder); + context_builder_options = {{Options::FILE_FORMAT, "orc"}, + {Options::MERGE_ENGINE, "deduplicate"}, + {"orc.read.enable-metrics", "true"}}; + + AddOptions(&context_builder, context_builder_options); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto meta1 = std::make_shared( "data-00e3ed53-16ba-4537-9264-b7dc03fefc65-0.orc", /*file_size=*/803, /*row_count=*/1, @@ -1391,7 +1511,8 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09/f1=10/bucket-1/", {meta1, meta2}); ASSERT_OK_AND_ASSIGN(auto data_split, builder.WithSnapshot(5).IsStreaming(false).RawConvertible(false).Build()); - auto internal_context = CreateInternalReadContext(read_context); + auto internal_context = + CreateInternalReadContext(read_context, read_context_load_options, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, {data_split})); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, ReadResultCollector::CollectResult(batch_reader.get())); diff --git a/src/paimon/core/operation/precise_file_index_read_test.cpp b/src/paimon/core/operation/precise_file_index_read_test.cpp index 640179ef9..faf139558 100644 --- a/src/paimon/core/operation/precise_file_index_read_test.cpp +++ b/src/paimon/core/operation/precise_file_index_read_test.cpp @@ -30,6 +30,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/file_index/bitmap/bitmap_file_index.h" #include "paimon/common/predicate/leaf_predicate_impl.h" @@ -145,30 +146,42 @@ class PreciseFileIndexReadTest : public ::testing::Test { Result> CreateSplitRead( const std::shared_ptr& read_context, + const TableLoadOptions& read_context_load_options, + const std::map& read_context_options, std::shared_ptr table_schema = nullptr) const { if (!table_schema) { SchemaManager schema_manager(std::make_shared(), - read_context->GetPath()); + read_context_load_options.path); PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(0)); } auto options = table_schema->Options(); - for (const auto& [key, value] : read_context->GetOptions()) { + for (const auto& [key, value] : read_context_options) { options[key] = value; } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, options, read_context_load_options.file_system, + read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, read_context_load_options.identifier, + read_context_options)); return std::make_unique(nullptr, context, pool_, nullptr); } Result> ReadWithFileIndex( - const std::shared_ptr& read_context, const std::shared_ptr& index, + const std::shared_ptr& read_context, + const TableLoadOptions& read_context_load_options, + const std::map& read_context_options, + const std::shared_ptr& index, const RoaringBitmap32& deleted_rows = RoaringBitmap32(), std::shared_ptr table_schema = nullptr, const std::optional>& row_ranges = std::nullopt, std::shared_ptr deletion_vector = nullptr, const std::optional& row_count = std::nullopt) const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_read, - CreateSplitRead(read_context, std::move(table_schema))); + CreateSplitRead(read_context, read_context_load_options, + read_context_options, std::move(table_schema))); auto file = PrepareDataFile(); file->embedded_index = index; if (row_count) { @@ -207,11 +220,16 @@ class PreciseFileIndexReadTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr index, MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, matches, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"f0"}).SetPredicate(predicate).EnablePreciseFileIndexEvaluate( true); + auto context_load_options = builder_load_options; + auto context_options = builder_options; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, builder.Finish()); - return ReadWithFileIndex(context, index); + return ReadWithFileIndex(context, context_load_options, context_options, index); } static void CheckSearchResult(const std::shared_ptr& actual, @@ -258,14 +276,18 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateNestedAndOrWithPartitionConst "f0", statistics_index_type_, {{first_query, {{{0, 0.2f}, {1, 0.4f}}, 0.2}}, {second_query, {{{2, 0.6f}}, 0.1}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}) - .SetPredicate(predicate) - .AddOption(Options::READ_BATCH_SIZE, "1"); + builder_options[Options::READ_BATCH_SIZE] = "1"; + builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index)); + ReadWithFileIndex(context, context_load_options, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}, @@ -298,12 +320,17 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateReordersQueries) { {{wide_query, {{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, 0.9}}, {narrow_query, {{{0, 0.4f}, {2, 0.5f}}, 0.1}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0", "f3"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index)); + ReadWithFileIndex(context, context_load_options, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, R"([["Tony",14.1]])"); @@ -449,15 +476,20 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateEmptyRowsShortCircuitExecutio {{empty_query, {{}, std::nullopt}}, {all_query, {{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, std::nullopt}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); for (bool all_deleted : {false, true}) { statistics_->executions.clear(); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadWithFileIndex(context, index, + ReadWithFileIndex(context, context_load_options, context_options, index, all_deleted ? RoaringBitmap32::From({0, 1, 2}) : RoaringBitmap32())); ASSERT_TRUE(!result || result->length() == 0); ASSERT_EQ(statistics_->executions.size(), all_deleted ? 0 : 1); @@ -507,15 +539,19 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateIndexedLeafSkipsUnselectedSca std::shared_ptr index, MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, {{query, {{{0, 0.25f}, {2, 0.75f}}, 0.1}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); // Single-row batches let bitmap selection skip the unselected scalar evaluation batch. - builder.SetReadFieldNames({"f0", "f3"}) - .SetPredicate(predicate) - .AddOption(Options::READ_BATCH_SIZE, "1"); + builder_options[Options::READ_BATCH_SIZE] = "1"; + builder.SetReadFieldNames({"f0", "f3"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index)); + ReadWithFileIndex(context, context_load_options, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, R"([["Tony",14.1]])"); @@ -552,14 +588,18 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRechecksScalarIndexCandidates std::shared_ptr predicate, PredicateBuilder::And( {PredicateBuilder::Equal(1, "f3", FieldType::DOUBLE, Literal(13.1)), text})); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}) - .SetPredicate(predicate) - .AddOption(Options::READ_BATCH_SIZE, "1"); + builder_options[Options::READ_BATCH_SIZE] = "1"; + builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index)); + ReadWithFileIndex(context, context_load_options, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), field, arrow::field("_f0_INDEX_SCORE", arrow::float32())}, @@ -578,14 +618,20 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicatePrechecksSearchBeforeEmptyCan PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(case_id == 0 ? 20 : 10)); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({scalar, text})); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); const auto deleted = case_id == 1 ? RoaringBitmap32::From({0, 1, 2}) : RoaringBitmap32(); const std::optional> ranges = case_id == 2 ? std::make_optional(std::vector()) : std::nullopt; - ASSERT_NOK_WITH_MSG(ReadWithFileIndex(context, nullptr, deleted, nullptr, ranges), + ASSERT_NOK_WITH_MSG(ReadWithFileIndex(context, context_load_options, context_options, + nullptr, deleted, nullptr, ranges), "has no File Index for search field 'f0'"); } } @@ -614,14 +660,22 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRenamedTextAndMissingScalarCo MockSearchFileIndex::MakeQueryIndex( "f0", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, 0.5}}}, pool_)); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) - .SetOptions(options) + builder_options = options; + builder + .SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) + .SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index, RoaringBitmap32(), current)); + ReadWithFileIndex(context, context_load_options, context_options, index, + RoaringBitmap32(), current)); CheckSearchResult( result, {arrow::field("renamed_f0", arrow::utf8()), arrow::field("new_value", arrow::int32()), @@ -632,14 +686,23 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRenamedTextAndMissingScalarCo ASSERT_OK_AND_ASSIGN( std::shared_ptr nonnull, PredicateBuilder::And({text, PredicateBuilder::IsNotNull(1, "new_value", FieldType::INT)})); - ReadContextBuilder nonnull_builder(SearchTablePath()); + TableLoadOptions nonnull_builder_load_options; + nonnull_builder_load_options.path = SearchTablePath(); + std::map nonnull_builder_options; + ReadContextBuilder nonnull_builder; nonnull_builder.EnablePreciseFileIndexEvaluate(true); - nonnull_builder.SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) - .SetOptions(options) + nonnull_builder_options = options; + nonnull_builder + .SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) + .SetPredicate(nonnull); + auto nonnull_context_load_options = nonnull_builder_load_options; + auto nonnull_context_options = nonnull_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr nonnull_context, nonnull_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr empty, - ReadWithFileIndex(nonnull_context, index, RoaringBitmap32(), current)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr empty, + ReadWithFileIndex(nonnull_context, nonnull_context_load_options, nonnull_context_options, + index, RoaringBitmap32(), current)); ASSERT_TRUE(!empty || empty->length() == 0); } @@ -666,19 +729,25 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateFullTextAndVectorScoresAfterF auto search = std::make_shared("f3", 1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); + builder_options[Options::READ_BATCH_SIZE] = "1"; builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0", "f3", "_f0_INDEX_SCORE"}) .SetPredicate(predicate) - .SetVectorSearch(search) - .AddOption(Options::READ_BATCH_SIZE, "1"); + .SetVectorSearch(search); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); for (int32_t case_id : {0, 1, 2}) { const auto deleted = case_id == 1 ? RoaringBitmap32::From({1}) : RoaringBitmap32(); const std::optional> ranges = case_id == 2 ? std::make_optional(std::vector{Range(2, 2)}) : std::nullopt; ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index, deleted, nullptr, ranges)); + ReadWithFileIndex(context, context_load_options, context_options, + index, deleted, nullptr, ranges)); CheckSearchResult( result, {arrow::field("_f3_INDEX_SCORE", arrow::float32()), arrow::field("f0", arrow::utf8()), @@ -691,21 +760,27 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateFullTextAndVectorScoresAfterF TEST_F(PreciseFileIndexReadTest, TestFullTextRootScoresAndLimitAfterScalarDvAndRanges) { const std::string query = R"({"match":{"query":"ranked"}})"; auto search = std::make_shared("f0", query, 1, RoaringBitmap64::From({0, 2})); - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); + builder_options[Options::READ_BATCH_SIZE] = "1"; builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}) .SetFullTextSearch(search) - .SetPredicate(PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.5))) - .AddOption(Options::READ_BATCH_SIZE, "1"); + .SetPredicate(PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.5))); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr index, MockSearchFileIndex::MakeQueryIndex( "f0", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, std::nullopt}}}, pool_)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, index, RoaringBitmap32::From({0}), nullptr, - std::vector{Range(1, 2)})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result, + ReadWithFileIndex(context, context_load_options, context_options, index, + RoaringBitmap32::From({0}), nullptr, std::vector{Range(1, 2)})); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64()), arrow::field("_f0_INDEX_SCORE", arrow::float32())}, @@ -729,7 +804,10 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec MockSearchFileIndex::Serialize(indexes, pool_)); for (const std::string& query_type : std::vector{"none", "scalar", "match", "full-text", "vector"}) { - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"f0", "f3"}).EnablePreciseFileIndexEvaluate(true); if (query_type == "scalar") { builder.SetPredicate( @@ -746,6 +824,8 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{})); } + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); for (bool empty_dv : {false, true}) { auto dv = std::make_shared(); @@ -756,7 +836,8 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec const std::optional> ranges = empty_candidates ? std::make_optional(std::vector()) : std::nullopt; ASSERT_NOK_WITH_MSG( - ReadWithFileIndex(context, index, RoaringBitmap32(), nullptr, ranges, dv), + ReadWithFileIndex(context, context_load_options, context_options, index, + RoaringBitmap32(), nullptr, ranges, dv), "Precise file-index evaluation requires bitmap32 deletion vectors"); } } @@ -766,7 +847,10 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Limit) { for (const std::string& search_type : std::vector{"none", "scalar", "match", "full-text", "vector"}) { - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0", "f3"}); if (search_type == "scalar") { @@ -785,9 +869,12 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Li "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{})); } + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_NOK_WITH_MSG( - ReadWithFileIndex(context, /*index=*/nullptr, RoaringBitmap32(), nullptr, + ReadWithFileIndex(context, context_load_options, context_options, /*index=*/nullptr, + RoaringBitmap32(), nullptr, /*row_ranges=*/std::nullopt, /*deletion_vector=*/nullptr, int64_t{RoaringBitmap32::MAX_VALUE} + 1), "Precise file-index evaluation requires bitmap32 physical row positions"); @@ -796,17 +883,22 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Li TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationAtBitmap32Limit) { for (bool has_predicate : {false, true}) { - ReadContextBuilder builder(SearchTablePath()); + TableLoadOptions builder_load_options; + builder_load_options.path = SearchTablePath(); + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"f0", "f3"}).EnablePreciseFileIndexEvaluate(true); if (has_predicate) { builder.SetPredicate( PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0))); } + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadWithFileIndex(context, /*index=*/nullptr, RoaringBitmap32(), nullptr, - std::vector{Range(0, 2)}, + ReadWithFileIndex(context, context_load_options, context_options, /*index=*/nullptr, + RoaringBitmap32(), nullptr, std::vector{Range(0, 2)}, /*deletion_vector=*/nullptr, RoaringBitmap32::MAX_VALUE)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index f259cf121..43da854fe 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -29,6 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/reader/concat_batch_reader.h" #include "paimon/common/types/data_field.h" @@ -198,15 +199,24 @@ class RawFileSplitReadTest : public ::testing::Test { } Result> ReadSearchWithDeletionVector( - const std::shared_ptr& read_context, const std::string& index_type, - const RoaringBitmap32& deleted_rows, std::shared_ptr table_schema) const { - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + const std::shared_ptr& read_context, + const TableLoadOptions& read_context_load_options, + const std::map& read_context_options, + const std::string& index_type, const RoaringBitmap32& deleted_rows, + std::shared_ptr table_schema) const { + SchemaManager schema_manager(std::make_shared(), + read_context_load_options.path); if (!table_schema) { PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(0)); } PAIMON_ASSIGN_OR_RAISE( std::unique_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, read_context_load_options.identifier, + read_context_options)); auto split = std::dynamic_pointer_cast(PrepareDataSplits().front()); if (!split) { return Status::Invalid("Raw search test split must be a DataSplitImpl"); @@ -225,7 +235,7 @@ class RawFileSplitReadTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( - read_context->GetPath(), schema, table_schema->PartitionKeys(), + read_context_load_options.path, schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, options.IndexFileInDataFileDir(), @@ -252,19 +262,31 @@ class RawFileSplitReadTest : public ::testing::Test { std::string path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/" "multi_partition_append_table"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames(read_schema->field_names()); if (dv_factory) { - context_builder.AddOption(Options::FILE_INDEX_READ_ENABLED, "true"); + context_builder_options[Options::FILE_INDEX_READ_ENABLED] = "true"; + context_builder.SetPredicate(PredicateBuilder::IsNotNull(0, "f0", FieldType::STRING)); } + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + read_context_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, read_context_load_options.identifier, + read_context_options)); auto data_splits = PrepareDataSplits(); const auto& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); @@ -343,17 +365,25 @@ TEST_F(RawFileSplitReadTest, TestCreateReader) { TEST_F(RawFileSplitReadTest, TestCreateReaderUsesExplicitPredicate) { const std::string path = GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; - ReadContextBuilder builder(path); - builder.EnablePredicateFilter(true) - .SetPredicate(PredicateBuilder::Equal(0, "f0", FieldType::STRING, - Literal(FieldType::STRING, "Bob", 3))) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false"); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; + builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + builder.EnablePredicateFilter(true).SetPredicate( + PredicateBuilder::Equal(0, "f0", FieldType::STRING, Literal(FieldType::STRING, "Bob", 3))); + auto read_context_load_options = builder_load_options; + auto read_context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); SchemaManager schemas(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, schemas.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); const CoreOptions& options = context->GetCoreOptions(); const auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, options.CreateExternalPaths()); @@ -606,17 +636,26 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { std::shared_ptr vector_search = std::make_shared( "f3", /*limit=*/2, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true); context_builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}) .SetVectorSearch(vector_search) .WithCache(cache); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN( - auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto internal_context, + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); auto data_splits = PrepareDataSplits(); auto data_split = std::dynamic_pointer_cast(data_splits.front()); @@ -695,14 +734,24 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { ASSERT_EQ(cache->GetCount(CacheKind::INDEX_READER), 4); ASSERT_EQ(cache->SupplierCallCount(CacheKind::INDEX_READER), 0); - ReadContextBuilder without_score_builder(path); + TableLoadOptions without_score_builder_load_options; + without_score_builder_load_options.path = path; + std::map without_score_builder_options; + ReadContextBuilder without_score_builder; without_score_builder.EnablePreciseFileIndexEvaluate(true); without_score_builder.SetReadFieldNames({"f0"}).SetVectorSearch(vector_search); + auto without_score_context_load_options = without_score_builder_load_options; + auto without_score_context_options = without_score_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr without_score_context, without_score_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto without_score_internal_context, - InternalReadContext::Create(without_score_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + without_score_context, table_schema, table_schema->Options(), + without_score_context_load_options.file_system, without_score_context_load_options.path, + without_score_context_load_options.fs_scheme_to_identifier_map, + without_score_context_load_options.catalog, + without_score_context_load_options.identifier, without_score_context_options)); auto without_score_split_read = std::make_unique( path_factory, std::move(without_score_internal_context), pool_, executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr without_score_reader, @@ -727,13 +776,19 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchFiltersDeletionVectorBeforeTopK) { std::shared_ptr search = std::make_shared( "f3", /*limit=*/1, std::vector{1.0f}, user_filter, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}).SetVectorSearch(search); + auto read_context_load_options = builder_load_options; + auto read_context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadSearchWithDeletionVector(read_context, MockSearchFileIndex::VECTOR_IDENTIFIER, + ReadSearchWithDeletionVector(read_context, read_context_load_options, read_context_options, + MockSearchFileIndex::VECTOR_IDENTIFIER, RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( @@ -753,13 +808,19 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchFiltersDeletionVectorBeforeLimit) user_filter.Add(2); auto search = std::make_shared("f3", R"({"match":{"query":"match"}})", /*limit=*/1, user_filter); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); + auto read_context_load_options = builder_load_options; + auto read_context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadSearchWithDeletionVector(read_context, MockSearchFileIndex::FULL_TEXT_IDENTIFIER, + ReadSearchWithDeletionVector(read_context, read_context_load_options, read_context_options, + MockSearchFileIndex::FULL_TEXT_IDENTIFIER, RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( @@ -799,11 +860,15 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { auto search = std::make_shared("renamed_f3", 2, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetOptions(options) - .SetReadFieldNames({"f0", "_renamed_f3_INDEX_SCORE"}) - .SetVectorSearch(search); + builder_options = options; + builder.SetReadFieldNames({"f0", "_renamed_f3_INDEX_SCORE"}).SetVectorSearch(search); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); auto expected_type = arrow::struct_( @@ -814,8 +879,9 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { {RoaringBitmap32::From({0}), R"([[0,"Tony",0.75]])"}}; for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); - ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, - current_schema)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector( + context, context_load_options, context_options, + identifier, deleted, current_schema)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); @@ -831,14 +897,20 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; - ReadContextBuilder missing_field_builder(path); + TableLoadOptions missing_field_builder_load_options; + missing_field_builder_load_options.path = path; + std::map missing_field_builder_options; + ReadContextBuilder missing_field_builder; missing_field_builder.EnablePreciseFileIndexEvaluate(true); - missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetVectorSearch( - missing_search); + missing_field_builder_options = options; + missing_field_builder.SetReadFieldNames({"f0"}).SetVectorSearch(missing_search); + auto missing_context_load_options = missing_field_builder_load_options; + auto missing_context_options = missing_field_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); - ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, identifier, RoaringBitmap32(), - current_schema), + ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, missing_context_load_options, + missing_context_options, identifier, + RoaringBitmap32(), current_schema), "has no matching field ID"); } @@ -870,9 +942,15 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) original_schema->PartitionKeys(), {}, options)); auto search = std::make_shared("renamed_f3", R"({"match":{"query":"match"}})", 2); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options; + builder_load_options.path = path; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); - builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch(search); + builder_options = options; + builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); auto expected_type = arrow::struct_( @@ -882,8 +960,9 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) {RoaringBitmap32::From({0}), R"([[0,"Tony"]])"}}; for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); - ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector(context, identifier, deleted, - current_schema)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector( + context, context_load_options, context_options, + identifier, deleted, current_schema)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); @@ -899,14 +978,20 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; - ReadContextBuilder missing_field_builder(path); + TableLoadOptions missing_field_builder_load_options; + missing_field_builder_load_options.path = path; + std::map missing_field_builder_options; + ReadContextBuilder missing_field_builder; missing_field_builder.EnablePreciseFileIndexEvaluate(true); - missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch( - missing_search); + missing_field_builder_options = options; + missing_field_builder.SetReadFieldNames({"f0"}).SetFullTextSearch(missing_search); + auto missing_context_load_options = missing_field_builder_load_options; + auto missing_context_options = missing_field_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); - ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, identifier, RoaringBitmap32(), - current_schema), + ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, missing_context_load_options, + missing_context_options, identifier, + RoaringBitmap32(), current_schema), "has no matching field ID"); } @@ -914,14 +999,24 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { std::string path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/" "multi_partition_append_table"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + read_context_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN( - auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto internal_context, + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); const auto& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, @@ -964,14 +1059,24 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { TEST_F(RawFileSplitReadTest, TestMatch) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = path; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + auto read_context_load_options = context_builder_load_options; + auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + SchemaManager schema_manager(std::make_shared(), + read_context_load_options.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr internal_context, + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_load_options.file_system, read_context_load_options.path, + read_context_load_options.fs_scheme_to_identifier_map, + read_context_load_options.catalog, + read_context_load_options.identifier, read_context_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique( diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index 36b5e8fc9..545099359 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -18,49 +18,33 @@ #include "paimon/read_context.h" -#include #include #include #include "arrow/c/abi.h" #include "arrow/c/bridge.h" -#include "paimon/catalog/catalog.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/table/system/system_table.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/predicate_utils.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; ReadContext::ReadContext( - const std::string& path, const std::string& branch, const std::vector& read_field_names, const std::vector& read_field_ids, const std::shared_ptr& predicate, const std::shared_ptr& vector_search, const std::shared_ptr& full_text_search, bool enable_precise_file_index_evaluate, bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, - uint32_t row_to_batch_thread_number, const std::optional& table_schema, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, const std::optional& identifier) - : path_(path), - branch_(branch), - read_field_names_(read_field_names), + uint32_t row_to_batch_thread_number, const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& realtime_context, bool read_ahead_cache_enabled, + const CacheConfig& cache_config, const std::shared_ptr& cache, WarmupLevel warmup_level) + : read_field_names_(read_field_names), read_field_ids_(read_field_ids), predicate_(predicate), vector_search_(vector_search), @@ -73,20 +57,13 @@ ReadContext::ReadContext( prefetch_max_parallel_num_(prefetch_max_parallel_num), enable_multi_thread_row_to_batch_(enable_multi_thread_row_to_batch), row_to_batch_thread_number_(row_to_batch_thread_number), - table_schema_(table_schema), memory_pool_(memory_pool), executor_(executor), - specific_file_system_(specific_file_system), - fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), - options_(options), read_ahead_cache_enabled_(read_ahead_cache_enabled), cache_config_(cache_config), cache_(cache), - format_table_(format_table), - warmup_level_(warmup_level), - catalog_(catalog), - identifier_(identifier) {} + warmup_level_(warmup_level) {} ReadContext::~ReadContext() { if (read_schema_ && read_schema_->release) { @@ -110,12 +87,9 @@ class ReadContextBuilder::Impl { public: friend class ReadContextBuilder; void Reset() { - branch_.reset(); read_field_names_.clear(); read_field_ids_.clear(); read_schema_.reset(); - fs_scheme_to_identifier_map_.clear(); - options_.clear(); predicate_.reset(); vector_search_.reset(); full_text_search_.reset(); @@ -128,32 +102,18 @@ class ReadContextBuilder::Impl { prefetch_max_parallel_num_ = 3; enable_multi_thread_row_to_batch_ = false; row_to_batch_thread_number_ = 1; - table_schema_ = std::nullopt; memory_pool_ = GetDefaultPool(); executor_.reset(); - specific_file_system_.reset(); realtime_context_.reset(); cache_config_ = CacheConfig(); cache_.reset(); warmup_level_ = WarmupLevel::RAW; - catalog_.reset(); - identifier_.reset(); } private: - std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one read of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; - /// Unset until `WithBranch()` names one, so that a branch named only by the `branch` - /// option is not overruled by the default. - std::optional branch_; std::vector read_field_names_; std::vector read_field_ids_; std::unique_ptr read_schema_; - std::map fs_scheme_to_identifier_map_; - std::map options_; std::shared_ptr predicate_; std::shared_ptr vector_search_; std::shared_ptr full_text_search_; @@ -165,49 +125,22 @@ class ReadContextBuilder::Impl { uint32_t prefetch_max_parallel_num_ = 3; bool enable_multi_thread_row_to_batch_ = false; uint32_t row_to_batch_thread_number_ = 1; - std::optional table_schema_; std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_; - std::shared_ptr specific_file_system_; std::shared_ptr realtime_context_; bool read_ahead_cache_enabled_ = true; CacheConfig cache_config_; std::shared_ptr cache_; WarmupLevel warmup_level_ = WarmupLevel::RAW; - std::shared_ptr catalog_; - std::optional identifier_; }; -ReadContextBuilder::ReadContextBuilder(const std::string& path) - : impl_(std::make_unique()) { - impl_->path_ = path; -} - -ReadContextBuilder::ReadContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} +ReadContextBuilder::ReadContextBuilder() : impl_(std::make_unique()) {} ReadContextBuilder::~ReadContextBuilder() = default; ReadContextBuilder::ReadContextBuilder(ReadContextBuilder&&) noexcept = default; ReadContextBuilder& ReadContextBuilder::operator=(ReadContextBuilder&&) noexcept = default; -ReadContextBuilder& ReadContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::SetOptions(const std::map& opts) { - impl_->options_ = opts; - return *this; -} - ReadContextBuilder& ReadContextBuilder::SetReadFieldNames( const std::vector& read_field_names) { impl_->read_field_names_ = read_field_names; @@ -301,35 +234,6 @@ ReadContextBuilder& ReadContextBuilder::WithRealtimeContext( return *this; } -ReadContextBuilder& ReadContextBuilder::SetTableSchema(const std::string& table_schema) { - impl_->table_schema_ = table_schema; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::WithBranch(const std::string& branch) { - impl_->branch_ = branch; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::WithFileSystemSchemeToIdentifierMap( - const std::map& fs_scheme_to_identifier_map) { - impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::WithFileSystem( - const std::shared_ptr& file_system) { - impl_->specific_file_system_ = file_system; - return *this; -} - -ReadContextBuilder& ReadContextBuilder::WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier) { - impl_->catalog_ = catalog; - impl_->identifier_.emplace(identifier); - return *this; -} - ReadContextBuilder& ReadContextBuilder::SetReadAheadCacheEnabled(bool enabled) { impl_->read_ahead_cache_enabled_ = enabled; return *this; @@ -379,81 +283,6 @@ Result> ReadContextBuilder::Finish() { return Status::Invalid("FullTextSearch field name and query must not be empty"); } } - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot read with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } - PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); - if (impl_->path_.empty()) { - return Status::Invalid("cannot read with empty table path"); - } - if (impl_->catalog_ == nullptr && impl_->identifier_) { - return Status::Invalid("cannot read through a null catalog"); - } - // Resolve a branch named by a system table path too, before requesting its credentials. - std::optional system_table_path; - if (impl_->format_table_ == nullptr) { - PAIMON_ASSIGN_OR_RAISE(system_table_path, SystemTableLoader::TryParsePath(impl_->path_)); - } - std::map branch_options = impl_->options_; - if (system_table_path && system_table_path->branch) { - PAIMON_ASSIGN_OR_RAISE(std::string path_branch, - BranchManager::ResolveBranch(impl_->identifier_, branch_options, - system_table_path->branch, "read")); - branch_options[Options::BRANCH] = path_branch; - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, branch_options, impl_->branch_, "read")); - if (impl_->catalog_ != nullptr && !BranchManager::IsMainBranch(branch)) { - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - } - std::optional identifier = impl_->identifier_; - if (identifier) { - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::ResolvedBranchIdentifier(identifier.value(), branch)); - identifier.emplace(std::move(branch_identifier)); - } - std::shared_ptr file_system = impl_->specific_file_system_; - if (impl_->catalog_ != nullptr && file_system == nullptr) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the read uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(file_system, - impl_->catalog_->GetTableFileSystem(identifier.value())); - } - std::map options = impl_->options_; - std::optional named_branch; - if (impl_->identifier_) { - PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); - } - if (impl_->branch_ || named_branch || options.count(Options::BRANCH) != 0 || - (system_table_path && system_table_path->branch)) { - options[Options::BRANCH] = branch; - } if (impl_->enable_prefetch_ && impl_->prefetch_max_parallel_num_ == 0) { return Status::Invalid("prefetch max parallel num should be greater than 0"); } @@ -465,10 +294,6 @@ Result> ReadContextBuilder::Finish() { return Status::Invalid( "prefetch batch count should be greater than or equal to prefetch max parallel num"); } - if (file_system && !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); - } if (!impl_->executor_) { // If the user do not set executor, create default executor by prefetch batch count uint32_t thread_count = impl_->enable_prefetch_ ? impl_->prefetch_max_parallel_num_ : 1; @@ -480,15 +305,14 @@ Result> ReadContextBuilder::Finish() { } auto ctx = std::make_unique( - impl_->path_, branch, impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, - impl_->vector_search_, impl_->full_text_search_, impl_->enable_precise_file_index_evaluate_, + impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, impl_->vector_search_, + impl_->full_text_search_, impl_->enable_precise_file_index_evaluate_, impl_->enable_predicate_filter_, impl_->enable_prefetch_, impl_->enable_late_materializing_, impl_->prefetch_batch_count_, impl_->prefetch_max_parallel_num_, impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_, - impl_->table_schema_, impl_->memory_pool_, impl_->executor_, file_system, - impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, options, - impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_, impl_->format_table_, - impl_->warmup_level_, impl_->catalog_, identifier); + impl_->memory_pool_, impl_->executor_, impl_->realtime_context_, + impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_, + impl_->warmup_level_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 5dabb79ef..2b5f53e57 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -24,6 +24,7 @@ #include "arrow/c/bridge.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/catalog_options.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/core/utils/branch_manager.h" @@ -41,205 +42,14 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(ReadContextTest, TestFailedFinishDoesNotRetainImplicitBranchOption) { - for (bool explicit_option : {false, true}) { - SCOPED_TRACE(explicit_option); - ReadContextBuilder builder("table_root_path"); - builder.WithBranch("dev").EnablePrefetch(true).SetPrefetchMaxParallelNum(0).AddOption( - "custom.option", "value"); - if (explicit_option) { - builder.AddOption(Options::BRANCH, "dev"); - } - ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num should be greater than 0"); - builder.WithBranch("main").SetPrefetchMaxParallelNum(1); - auto retried = builder.Finish(); - if (explicit_option) { - ASSERT_NOK_WITH_MSG(retried, "but both 'dev' and 'main' were named"); - } else { - ASSERT_OK(retried); - ASSERT_EQ("main", retried.value()->GetBranch()); - ASSERT_EQ("main", retried.value()->GetOptions().at(Options::BRANCH)); - ASSERT_EQ("value", retried.value()->GetOptions().at("custom.option")); - } - } -} - -TEST(ReadContextTest, TestWithCatalogResolvesTableFileSystem) { - // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context, which also keeps the - // catalog and identifier for the operators to resolve the table through. - auto table_fs = std::make_shared(); - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - - ReadContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); - - // Finish() resets the builder, so the next context reads without the catalog. - ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); - ASSERT_FALSE(next_ctx->GetCatalog()); - ASSERT_FALSE(next_ctx->GetIdentifier()); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); -} - -TEST(ReadContextTest, TestWithCatalogResolvesTheBranch) { - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - - for (const std::string branch : {"MAIN", "Main"}) { - const std::string message = - "a catalog names branch '" + branch + "' as it names the main branch"; - ASSERT_NOK_WITH_MSG(ReadContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, branch) - .Finish(), - message); - ASSERT_NOK_WITH_MSG(ReadContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1")) - .WithBranch(branch) - .Finish(), - message); - } - ASSERT_NOK_WITH_MSG(ReadContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1")) - .WithBranch("dev$options") - .Finish(), - "a branch a catalog addresses cannot contain '$'"); - ASSERT_NOK_WITH_MSG(ReadContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1", "dev")) - .AddOption(Options::BRANCH, "test") - .Finish(), - "a read is aimed at one branch, but both 'dev' and 'test' were named"); - ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); - ASSERT_EQ(0u, catalog->LoadTableSchemaCalls()); - ASSERT_OK_AND_ASSIGN(auto path_ctx, - ReadContextBuilder("table_root_path").WithBranch("MAIN").Finish()); - ASSERT_EQ("MAIN", path_ctx->GetBranch()); - - ASSERT_OK_AND_ASSIGN(auto option_ctx, ReadContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ("dev", option_ctx->GetBranch()); - ASSERT_EQ(Identifier("db1", "t1$branch_dev"), option_ctx->GetIdentifier().value()); - ASSERT_OK_AND_ASSIGN(auto branch_ctx, ReadContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1")) - .WithBranch("dev") - .Finish()); - ASSERT_EQ(Identifier("db1", "t1$branch_dev"), branch_ctx->GetIdentifier().value()); - ASSERT_EQ(std::vector( - {Identifier("db1", "t1$branch_dev"), Identifier("db1", "t1$branch_dev")}), - catalog->TableFileSystemRequests()); - for (const std::string branch : {"MAIN", "Main"}) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1$branch_" + branch)) - .Finish()); - ASSERT_EQ("main", context->GetBranch()); - ASSERT_OK_AND_ASSIGN(std::string resolved_branch, - context->GetIdentifier()->GetBranchNameOrDefault()); - ASSERT_EQ("main", resolved_branch); - ASSERT_EQ(context->GetIdentifier().value(), catalog->TableFileSystemRequests().back()); - } -} - -TEST(ReadContextTest, TestSystemTablePathResolvesBranchBeforeCatalogRequest) { - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - const std::string path = "table_root_path$branch_dev$files"; - for (const std::string table : {"t1", "t1$files"}) { - ASSERT_OK_AND_ASSIGN( - std::unique_ptr context, - ReadContextBuilder(path).WithCatalog(catalog, Identifier("db1", table)).Finish()); - ASSERT_EQ("dev", context->GetBranch()); - ASSERT_EQ(Identifier("db1", table == "t1" ? "t1$branch_dev" : "t1$branch_dev$files"), - context->GetIdentifier().value()); - ASSERT_EQ(context->GetIdentifier().value(), catalog->TableFileSystemRequests().back()); - } - const size_t requests = catalog->TableFileSystemRequests().size(); - ASSERT_NOK_WITH_MSG( - ReadContextBuilder(path).WithCatalog(catalog, Identifier("db1", "t1", "other")).Finish(), - "a read is aimed at one branch, but both 'other' and 'dev' were named"); - ASSERT_NOK_WITH_MSG(ReadContextBuilder(path) - .WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "main") - .Finish(), - "a read is aimed at one branch, but both 'main' and 'dev' were named"); - ASSERT_NOK_WITH_MSG(ReadContextBuilder(path) - .WithCatalog(catalog, Identifier("db1", "t1")) - .WithBranch("main") - .Finish(), - "a read is aimed at one branch, but both 'dev' and 'main' were named"); - ASSERT_NOK_WITH_MSG(ReadContextBuilder("table_root_path$branch_MAIN$files") - .WithCatalog(catalog, Identifier("db1", "t1")) - .Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - ASSERT_EQ(requests, catalog->TableFileSystemRequests().size()); -} - -TEST(ReadContextTest, TestWithFileSystemOverridesCatalog) { - // A file system the caller gave is the one that is used, and the catalog is not asked for one. - auto table_fs = std::make_shared(); - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - auto given_fs = std::make_shared(); - - ReadContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN( - auto ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), given_fs); - ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); -} - -TEST(ReadContextTest, TestWithCatalogNullRejected) { - ReadContextBuilder builder("table_root_path"); - ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot read through a null catalog"); -} - -TEST(ReadContextTest, TestWithCatalogKeepsReadOptionsSeparate) { - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - catalog->SetOptions({{CatalogOptions::METASTORE, "rest"}, - {CatalogOptions::URI, "http://catalog"}, - {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}}); - - ReadContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - // Finishing a read neither imports catalog defaults nor generates a dependency header. - ASSERT_TRUE(ctx->GetOptions().empty()); - - catalog->SetOptions({{CatalogOptions::METASTORE, "rest"}, - {CatalogOptions::URI, "http://catalog"}, - {"header.X-Paimon-Read-Via", "catalog-view"}}); - ASSERT_OK_AND_ASSIGN(auto nested_ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_TRUE(nested_ctx->GetOptions().empty()); - const std::map read_options = { - {Options::BLOB_VIEW_RESOLVE_ENABLED, "true"}, - {"header.X-Paimon-Read-Via", "explicit-view"}}; - ASSERT_OK_AND_ASSIGN( - auto explicit_ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).SetOptions(read_options).Finish()); - ASSERT_EQ(explicit_ctx->GetOptions(), read_options); -} TEST(ReadContextTest, TestDefaultValue) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetReadFieldNames().empty()); ASSERT_TRUE(ctx->GetReadFieldIds().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetPredicate()); ASSERT_FALSE(ctx->HasVectorOrFullTextSearch()); ASSERT_FALSE(ctx->EnablePreciseFileIndexEvaluate()); @@ -251,13 +61,10 @@ TEST(ReadContextTest, TestDefaultValue) { ASSERT_EQ(3, ctx->GetPrefetchMaxParallelNum()); ASSERT_FALSE(ctx->EnableMultiThreadRowToBatch()); ASSERT_EQ(1, ctx->GetRowToBatchThreadNumber()); - ASSERT_EQ("main", ctx->GetBranch()); - ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); } TEST(ReadContextTest, TestSetContent) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; std::shared_ptr memory_pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); CacheConfig cache_config; @@ -265,7 +72,6 @@ TEST(ReadContextTest, TestSetContent) { cache_config.SetHoleSizeLimit(128); cache_config.SetPreBufferLimit(2048); - builder.AddOption("key", "value"); builder.SetReadFieldNames({"f1", "f2"}); builder.SetReadFieldIds({0, 1}); auto predicate = @@ -282,17 +88,15 @@ TEST(ReadContextTest, TestSetContent) { builder.SetRowToBatchThreadNumber(9); builder.WithMemoryPool(memory_pool); builder.WithExecutor(executor); - builder.SetTableSchema("table-schema-json"); - builder.WithBranch("rt"); + builder.WithCacheConfig(cache_config); auto fs = std::make_shared(); - builder.WithFileSystem(fs); + auto manifest_cache = std::make_shared(1024); builder.WithCache(manifest_cache); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); // test result - ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_EQ(ctx->GetReadFieldNames(), std::vector({"f1", "f2"})); @@ -309,23 +113,15 @@ TEST(ReadContextTest, TestSetContent) { ASSERT_EQ(9, ctx->GetRowToBatchThreadNumber()); ASSERT_EQ(memory_pool, ctx->GetMemoryPool()); ASSERT_EQ(executor, ctx->GetExecutor()); - ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); - ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); - ASSERT_EQ("rt", ctx->GetBranch()); ASSERT_EQ(512U, ctx->GetCacheConfig().GetRangeSizeLimit()); ASSERT_EQ(128U, ctx->GetCacheConfig().GetHoleSizeLimit()); ASSERT_EQ(2048U, ctx->GetCacheConfig().GetPreBufferLimit()); - ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); - std::map expected_options = {{"key", "value"}, - {Options::BRANCH, "rt"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); ASSERT_TRUE(ctx->GetCache()); } TEST(ReadContextTest, TestSetWarmupLevel) { for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; // The setter hands back the builder so it chains like every other setter on it. ASSERT_EQ(&builder, &builder.SetWarmupLevel(level)); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -333,7 +129,7 @@ TEST(ReadContextTest, TestSetWarmupLevel) { } // Finish() resets the builder, so reusing one must not carry the previous warmup level over. - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.SetWarmupLevel(WarmupLevel::NONE); ASSERT_OK_AND_ASSIGN(auto first_ctx, builder.Finish()); ASSERT_EQ(WarmupLevel::NONE, first_ctx->GetWarmupLevel()); @@ -356,7 +152,7 @@ TEST(ReadContextTest, TestSearchRequiresPreciseFileIndexEvaluate) { PredicateBuilder::And({scalar, disjunction})); for (int32_t search_type = 0; search_type < 3; ++search_type) { for (bool enable_before_search : {false, true}) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(enable_before_search); if (search_type == 0) { builder.SetVectorSearch(vector); @@ -366,7 +162,6 @@ TEST(ReadContextTest, TestSearchRequiresPreciseFileIndexEvaluate) { builder.SetPredicate(predicate); } if (!enable_before_search) { - ASSERT_NOK_WITH_MSG(builder.Finish(), "EnablePreciseFileIndexEvaluate(true)"); builder.EnablePreciseFileIndexEvaluate(true); } ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); @@ -375,97 +170,10 @@ TEST(ReadContextTest, TestSearchRequiresPreciseFileIndexEvaluate) { } } -TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) { - ReadContextBuilder builder("table_root_path"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - -TEST(ReadContextTest, TestBranch) { - ReadContextBuilder option_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto option_ctx, option_builder.AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", option_ctx->GetBranch()); - - ReadContextBuilder agreeing_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN( - auto agreeing_ctx, - agreeing_builder.WithBranch("rt").AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", agreeing_ctx->GetBranch()); - - ReadContextBuilder mixed_builder("table_root_path"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithBranch("rt").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'rt' were named"); - - // An empty branch selects the main branch and stays accepted. - ReadContextBuilder main_builder("table_root_path"); - main_builder.WithBranch(""); - ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish()); - ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch()); - ReadContextBuilder empty_builder("table_root_path"); - ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'main' were named"); - - // The branch names a directory under the table path, so a value that is not a single path - // component is rejected when the context is built. - ReadContextBuilder escaping_builder("table_root_path"); - escaping_builder.WithBranch("rt/../../../../../outside"); - ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain path separators"); -} - -TEST(ReadContextTest, TestBranchAgreesWithIdentifier) { - // A read through a catalog names its table with an Identifier that can itself carry a branch, - // so that branch and the one WithBranch() names must agree, exactly as the write and commit - // builders already require: one read cannot be aimed at two branches at once. - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - - // The identifier's branch is honored when WithBranch() stays silent. - ReadContextBuilder identifier_builder("table_root_path"); - identifier_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")); - ASSERT_OK_AND_ASSIGN(auto identifier_ctx, identifier_builder.Finish()); - ASSERT_EQ("dev", identifier_ctx->GetBranch()); - - // An agreeing WithBranch() is accepted. - ReadContextBuilder agreeing_builder("table_root_path"); - agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).WithBranch("dev"); - ASSERT_OK_AND_ASSIGN(auto agreeing_ctx, agreeing_builder.Finish()); - ASSERT_EQ("dev", agreeing_ctx->GetBranch()); - - // A disagreeing WithBranch() is refused, and both branches are named. - ReadContextBuilder mixed_builder("table_root_path"); - mixed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).WithBranch("rt"); - ASSERT_NOK_WITH_MSG(mixed_builder.Finish(), "but both 'dev' and 'rt' were named"); -} - -TEST(ReadContextTest, TestFileSystemAndSchemeMapConflict) { - ReadContextBuilder builder("table_root_path"); - auto fs = std::make_shared(); - builder.WithFileSystem(fs); - builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); - ASSERT_NOK_WITH_MSG( - builder.Finish(), - "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); -} - -TEST(ReadContextTest, TestSchemeMapWithoutFileSystem) { - ReadContextBuilder builder("table_root_path"); - builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - std::map expected_fs_map = {{"file", "local"}}; - ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); -} - TEST(ReadContextTest, TestPrefetchMaxParallelNumZero) { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.EnablePrefetch(true); builder.SetPrefetchMaxParallelNum(0); - ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num should be greater than 0"); } TEST(ReadContextTest, TestSetReadSchemaAndHasReadSchema) { @@ -475,7 +183,7 @@ TEST(ReadContextTest, TestSetReadSchemaAndHasReadSchema) { ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); { - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_TRUE(ctx->HasReadSchema()); @@ -488,7 +196,7 @@ TEST(ReadContextTest, TestSetReadSchemaAndHasReadSchema) { TEST(ReadContextTest, TestSetInvalidReadSchemaIgnored) { auto invalid_schema = std::make_unique(); - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.SetReadSchema(std::move(invalid_schema)); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -500,7 +208,7 @@ TEST(ReadContextTest, TestFileIndexSearchConfigurationAndReset) { std::shared_ptr vector_search = std::make_shared( "embedding", /*limit=*/3, std::vector{1.0f, 2.0f}, nullptr, nullptr, VectorSearch::DistanceType::COSINE, std::map{}); - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; ASSERT_EQ(&builder, &builder.SetVectorSearch(vector_search)); builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, builder.Finish()); @@ -529,7 +237,7 @@ TEST(ReadContextTest, TestRejectConflictingFileIndexSearch) { nullptr, std::nullopt, std::map{}); std::shared_ptr full_text_search = std::make_shared("body", R"({"match":{"query":"paimon"}})", /*limit=*/5); - ReadContextBuilder conflicting("table_root_path"); + ReadContextBuilder conflicting; conflicting.EnablePreciseFileIndexEvaluate(true) .SetVectorSearch(vector_search) .SetFullTextSearch(full_text_search); @@ -545,7 +253,7 @@ TEST(ReadContextTest, TestFullTextPredicateAndRootTopKConfiguration) { auto vector = std::make_shared("embedding", 3, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder builder("table_root_path"); + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"body", "price"}) .SetPredicate(filter) @@ -566,11 +274,11 @@ TEST(ReadContextTest, TestRootTopKValidation) { auto vector = std::make_shared("embedding", 0, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder vector_builder("table_root_path"); + ReadContextBuilder vector_builder; vector_builder.EnablePreciseFileIndexEvaluate(true).SetVectorSearch(vector); ASSERT_NOK_WITH_MSG(vector_builder.Finish(), "VectorSearch TopK limit must be positive"); auto text = std::make_shared("body", R"({"match":{"query":"apple"}})", -1); - ReadContextBuilder text_builder("table_root_path"); + ReadContextBuilder text_builder; text_builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(text); ASSERT_NOK_WITH_MSG(text_builder.Finish(), "FullTextSearch TopK limit must be positive"); } diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 944b0c6d0..1c261eecf 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -21,63 +21,29 @@ #include #include -#include "paimon/catalog/catalog.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/table/system/system_table.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; -ScanContext::ScanContext( - const std::string& path, bool is_streaming_mode, std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier) - : ScanContext(path, is_streaming_mode, limit, scan_filter, global_index_result, - realtime_context, memory_pool, executor, specific_file_system, table_schema, - options, cache, format_table, catalog, identifier, - /*read_field_names=*/std::nullopt) {} - -ScanContext::ScanContext( - const std::string& path, bool is_streaming_mode, std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier, - const std::optional>& read_field_names) - : path_(path), - is_streaming_mode_(is_streaming_mode), +ScanContext::ScanContext(bool is_streaming_mode, std::optional limit, + const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, + const std::shared_ptr& realtime_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& cache, + const std::optional>& read_field_names) + : is_streaming_mode_(is_streaming_mode), limit_(limit), scan_filters_(scan_filter), global_index_result_(global_index_result), realtime_context_(realtime_context), memory_pool_(memory_pool), executor_(executor), - specific_file_system_(specific_file_system), - table_schema_(table_schema), - options_(options), cache_(cache), - format_table_(format_table), - catalog_(catalog), - identifier_(identifier), read_field_names_(read_field_names) {} ScanContext::~ScanContext() = default; @@ -96,21 +62,11 @@ class ScanContextBuilder::Impl { realtime_context_.reset(); memory_pool_ = GetDefaultPool(); executor_.reset(); - specific_file_system_.reset(); - table_schema_ = std::nullopt; - options_.clear(); cache_.reset(); - catalog_.reset(); - identifier_.reset(); read_field_names_.reset(); } private: - std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one scan of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; bool is_streaming_mode_ = false; std::optional limit_; std::optional bucket_filter_; @@ -121,28 +77,11 @@ class ScanContextBuilder::Impl { std::shared_ptr memory_pool_ = GetDefaultPool(); // Resolved in Finish(); a builder never owns an executor of its own. std::shared_ptr executor_; - std::shared_ptr specific_file_system_; - std::optional table_schema_; - std::map options_; std::shared_ptr cache_; - std::shared_ptr catalog_; - std::optional identifier_; std::optional> read_field_names_; }; -ScanContextBuilder::ScanContextBuilder(const std::string& path) - : impl_(std::make_unique()) { - impl_->path_ = path; -} - -ScanContextBuilder::ScanContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} +ScanContextBuilder::ScanContextBuilder() : impl_(std::make_unique()) {} ScanContextBuilder::~ScanContextBuilder() = default; ScanContextBuilder& ScanContextBuilder::WithStreamingMode(bool is_streaming_mode) { @@ -166,6 +105,12 @@ ScanContextBuilder& ScanContextBuilder::SetPartitionFilter( return *this; } +ScanContextBuilder& ScanContextBuilder::SetReadFieldNames( + const std::vector& read_field_names) { + impl_->read_field_names_ = read_field_names; + return *this; +} + ScanContextBuilder& ScanContextBuilder::SetPredicate(const std::shared_ptr& predicate) { impl_->predicates_ = predicate; return *this; @@ -183,18 +128,6 @@ ScanContextBuilder& ScanContextBuilder::WithRealtimeContext( return *this; } -ScanContextBuilder& ScanContextBuilder::SetOptions( - const std::map& options) { - impl_->options_ = options; - return *this; -} - -ScanContextBuilder& ScanContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - ScanContextBuilder& ScanContextBuilder::WithMemoryPool( const std::shared_ptr& memory_pool) { impl_->memory_pool_ = memory_pool; @@ -206,108 +139,20 @@ ScanContextBuilder& ScanContextBuilder::WithExecutor(const std::shared_ptr& file_system) { - impl_->specific_file_system_ = file_system; - return *this; -} - -ScanContextBuilder& ScanContextBuilder::WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier) { - impl_->catalog_ = catalog; - impl_->identifier_.emplace(identifier); - return *this; -} - -ScanContextBuilder& ScanContextBuilder::SetReadFieldNames( - const std::vector& read_field_names) { - impl_->read_field_names_ = read_field_names; - return *this; -} - -ScanContextBuilder& ScanContextBuilder::SetTableSchema(const std::string& table_schema) { - impl_->table_schema_ = table_schema; - return *this; -} - ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr& cache) { impl_->cache_ = cache; return *this; } Result> ScanContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot scan with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers both, and from a source this cannot see behind, so a second - // answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } - } - PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); - if (impl_->path_.empty()) { - return Status::Invalid("cannot scan with empty table path"); - } - if (impl_->catalog_ == nullptr && impl_->identifier_) { - return Status::Invalid("cannot scan through a null catalog"); - } - // Resolve a branch named by a system table path too, before requesting its credentials. - std::optional system_table_path; - if (impl_->format_table_ == nullptr) { - PAIMON_ASSIGN_OR_RAISE(system_table_path, SystemTableLoader::TryParsePath(impl_->path_)); - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, - system_table_path ? system_table_path->branch : std::nullopt, - "scan")); - if (impl_->catalog_ != nullptr && !BranchManager::IsMainBranch(branch)) { - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - } - std::map options = impl_->options_; - std::optional named_branch; - if (impl_->identifier_) { - PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); - } - if (named_branch || options.count(Options::BRANCH) != 0 || - (system_table_path && system_table_path->branch)) { - options[Options::BRANCH] = branch; - } - std::optional identifier = impl_->identifier_; - if (identifier) { - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::ResolvedBranchIdentifier(identifier.value(), branch)); - identifier.emplace(std::move(branch_identifier)); - } - // Keep derived credentials local so a failed Finish() cannot bind a later retry to a table. - std::shared_ptr file_system = impl_->specific_file_system_; - if (impl_->catalog_ != nullptr && file_system == nullptr) { - PAIMON_ASSIGN_OR_RAISE(file_system, - impl_->catalog_->GetTableFileSystem(identifier.value())); - } std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); auto ctx = std::make_unique( - impl_->path_, impl_->is_streaming_mode_, impl_->limit_, + impl_->is_streaming_mode_, impl_->limit_, std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, - file_system, impl_->table_schema_, options, impl_->cache_, impl_->format_table_, - impl_->catalog_, identifier, impl_->read_field_names_); + impl_->cache_, impl_->read_field_names_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 5be2cb8c1..343917eff 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -19,6 +19,7 @@ #include "paimon/scan_context.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -31,157 +32,10 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(ScanContextTest, TestWithCatalogResolvesTableFileSystem) { - // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context, which also keeps the - // catalog and identifier for the operators to resolve the table through. - auto table_fs = std::make_shared(); - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - - ScanContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); - - // Finish() resets the builder, so the next context scans without the catalog. - ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); - ASSERT_FALSE(next_ctx->GetCatalog()); - ASSERT_FALSE(next_ctx->GetIdentifier()); - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); -} - -TEST(ScanContextTest, TestWithFileSystemOverridesCatalog) { - // A file system the caller gave is the one that is used, and the catalog is not asked for one. - auto table_fs = std::make_shared(); - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - auto given_fs = std::make_shared(); - - ScanContextBuilder builder("table_root_path"); - ASSERT_OK_AND_ASSIGN( - auto ctx, - builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); - ASSERT_EQ(ctx->GetSpecificFileSystem(), given_fs); - ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); -} - -TEST(ScanContextTest, TestWithCatalogResolvesTheBranch) { - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - - for (const std::string branch : {"MAIN", "Main"}) { - const std::string message = - "a catalog names branch '" + branch + "' as it names the main branch"; - ASSERT_NOK_WITH_MSG(ScanContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, branch) - .Finish(), - message); - } - ASSERT_NOK_WITH_MSG(ScanContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev$options") - .Finish(), - "a branch a catalog addresses cannot contain '$'"); - ASSERT_NOK_WITH_MSG(ScanContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1", "dev")) - .AddOption(Options::BRANCH, "test") - .Finish(), - "a scan is aimed at one branch, but both 'dev' and 'test' were named"); - ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); - ASSERT_EQ(0u, catalog->LoadTableSchemaCalls()); - ASSERT_OK_AND_ASSIGN( - auto path_ctx, - ScanContextBuilder("table_root_path").AddOption(Options::BRANCH, "MAIN").Finish()); - ASSERT_EQ("MAIN", path_ctx->GetOptions().at(Options::BRANCH)); - - ASSERT_OK_AND_ASSIGN(auto ctx, ScanContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1", "dev")) - .Finish()); - ASSERT_EQ("dev", ctx->GetOptions().at(Options::BRANCH)); - ASSERT_OK_AND_ASSIGN(auto option_ctx, ScanContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ("dev", option_ctx->GetOptions().at(Options::BRANCH)); - ASSERT_EQ(Identifier("db1", "t1$branch_dev"), option_ctx->GetIdentifier().value()); - ASSERT_OK_AND_ASSIGN(auto system_ctx, ScanContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1$files")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ(Identifier("db1", "t1$branch_dev$files"), system_ctx->GetIdentifier().value()); - ASSERT_OK_AND_ASSIGN(auto sys_ctx, ScanContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("sys", "tables")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ(Identifier("sys", "tables"), sys_ctx->GetIdentifier().value()); - ASSERT_OK_AND_ASSIGN(auto main_ctx, ScanContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1")) - .Finish()); - ASSERT_EQ(0u, main_ctx->GetOptions().count(Options::BRANCH)); - ASSERT_EQ(Identifier("db1", "t1"), main_ctx->GetIdentifier().value()); - ASSERT_EQ(std::vector({Identifier("db1", "t1$branch_dev"), - Identifier("db1", "t1$branch_dev"), - Identifier("db1", "t1$branch_dev$files"), - Identifier("sys", "tables"), Identifier("db1", "t1")}), - catalog->TableFileSystemRequests()); - for (const std::string branch : {"MAIN", "Main"}) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ScanContextBuilder("table_root_path") - .WithCatalog(catalog, Identifier("db1", "t1$branch_" + branch)) - .Finish()); - ASSERT_EQ("main", context->GetOptions().at(Options::BRANCH)); - ASSERT_OK_AND_ASSIGN(std::string resolved_branch, - context->GetIdentifier()->GetBranchNameOrDefault()); - ASSERT_EQ("main", resolved_branch); - ASSERT_EQ(context->GetIdentifier().value(), catalog->TableFileSystemRequests().back()); - } -} - -TEST(ScanContextTest, TestSystemTablePathResolvesBranchBeforeCatalogRequest) { - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - const std::string path = "table_root_path$branch_dev$files"; - for (const std::string table : {"t1", "t1$files"}) { - ASSERT_OK_AND_ASSIGN( - std::unique_ptr context, - ScanContextBuilder(path).WithCatalog(catalog, Identifier("db1", table)).Finish()); - ASSERT_EQ("dev", context->GetOptions().at(Options::BRANCH)); - ASSERT_EQ(Identifier("db1", table == "t1" ? "t1$branch_dev" : "t1$branch_dev$files"), - context->GetIdentifier().value()); - ASSERT_EQ(context->GetIdentifier().value(), catalog->TableFileSystemRequests().back()); - } - const size_t requests = catalog->TableFileSystemRequests().size(); - ASSERT_NOK_WITH_MSG( - ScanContextBuilder(path).WithCatalog(catalog, Identifier("db1", "t1", "other")).Finish(), - "a scan is aimed at one branch, but both 'other' and 'dev' were named"); - ASSERT_NOK_WITH_MSG(ScanContextBuilder(path) - .WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "main") - .Finish(), - "a scan is aimed at one branch, but both 'main' and 'dev' were named"); - ASSERT_NOK_WITH_MSG(ScanContextBuilder("table_root_path$branch_MAIN$files") - .WithCatalog(catalog, Identifier("db1", "t1")) - .Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - ASSERT_EQ(requests, catalog->TableFileSystemRequests().size()); -} - -TEST(ScanContextTest, TestWithCatalogNullRejected) { - ScanContextBuilder builder("table_root_path"); - ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot scan through a null catalog"); -} TEST(ScanContextTest, TestDefaultValue) { - ScanContextBuilder builder("table_root_path"); + ScanContextBuilder builder; ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_FALSE(ctx->IsStreamingMode()); ASSERT_FALSE(ctx->GetLimit()); ASSERT_TRUE(ctx->GetMemoryPool()); @@ -190,14 +44,11 @@ TEST(ScanContextTest, TestDefaultValue) { ASSERT_FALSE(ctx->GetScanFilters()->GetBucketFilter()); ASSERT_FALSE(ctx->GetScanFilters()->GetPredicate()); ASSERT_TRUE(ctx->GetScanFilters()->GetPartitionFilters().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetGlobalIndexResult()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); - ASSERT_FALSE(ctx->GetReadFieldNames()); } TEST(ScanContextTest, TestSetContent) { - ScanContextBuilder builder("table_root_path"); + ScanContextBuilder builder; std::shared_ptr memory_pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); @@ -211,18 +62,16 @@ TEST(ScanContextTest, TestSetContent) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); builder.SetGlobalIndexResult(global_index_result); builder.SetLimit(1000); - builder.AddOption("key", "value"); + builder.WithStreamingMode(true); builder.WithMemoryPool(memory_pool); builder.WithExecutor(executor); auto fs = std::make_shared(); - builder.WithFileSystem(fs); - builder.SetTableSchema("table-schema-json"); + auto manifest_cache = std::make_shared(1024); builder.WithCache(manifest_cache); builder.SetReadFieldNames({"f1", "f2"}); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_TRUE(ctx->IsStreamingMode()); ASSERT_EQ(1000, ctx->GetLimit()); ASSERT_TRUE(ctx->GetScanFilters()); @@ -232,33 +81,17 @@ TEST(ScanContextTest, TestSetContent) { ASSERT_EQ("{1,2,4,5}", ctx->GetGlobalIndexResult()->ToString()); ASSERT_EQ(memory_pool, ctx->GetMemoryPool()); ASSERT_EQ(executor, ctx->GetExecutor()); - ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); - ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); - std::map expected_options = {{"key", "value"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); - ASSERT_EQ(fs, ctx->GetSpecificFileSystem()); ASSERT_TRUE(ctx->GetCache()); ASSERT_EQ(std::optional>(std::vector{"f1", "f2"}), ctx->GetReadFieldNames()); } -TEST(ScanContextTest, TestSetOptionsOverridesAddedOptions) { - ScanContextBuilder builder("table_root_path"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - TEST(ScanContextTest, TestDefaultExecutorIsCreatedPerContext) { // A builder without WithExecutor() gives every context a default executor // of its own; nothing is shared across contexts. - ScanContextBuilder first_builder("table_root_path"); + ScanContextBuilder first_builder; ASSERT_OK_AND_ASSIGN(auto first_ctx, first_builder.Finish()); - ScanContextBuilder second_builder("table_root_path"); + ScanContextBuilder second_builder; ASSERT_OK_AND_ASSIGN(auto second_ctx, second_builder.Finish()); ASSERT_TRUE(first_ctx->GetExecutor()); ASSERT_TRUE(second_ctx->GetExecutor()); @@ -278,20 +111,4 @@ TEST(ScanContextTest, TestDefaultExecutorIsCreatedPerContext) { ASSERT_NE(executor, reset_ctx->GetExecutor()); } -TEST(ScanContextTest, FailedBranchValidationDoesNotRetainTableCredentials) { - auto first_fs = std::make_shared(); - auto second_fs = std::make_shared(); - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(first_fs); - ScanContextBuilder builder("table_b"); - builder.WithCatalog(catalog, Identifier("db", "a", "dev")) - .SetOptions({{Options::BRANCH, "main"}}); - ASSERT_NOK(builder.Finish()); - catalog->SetTableFileSystem(second_fs); - builder.WithCatalog(catalog, Identifier("db", "b")); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_EQ(second_fs, context->GetSpecificFileSystem()); - ASSERT_EQ(Identifier("db", "b"), catalog->TableFileSystemRequests().back()); -} - } // namespace paimon::test diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index 2fdf0dab2..a18f9a7cf 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -18,39 +18,26 @@ #include "paimon/write_context.h" -#include #include #include "arrow/util/thread_pool.h" -#include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { -WriteContext::WriteContext( - const std::string& root_path, const std::string& commit_user, bool is_streaming_mode, - bool ignore_num_bucket_check, bool ignore_previous_files, bool enable_multi_thread_spill, - const std::optional& write_id, const std::string& branch, - const std::vector& write_schema, const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, const std::string& temp_directory, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier) - : root_path_(root_path), - commit_user_(commit_user), - branch_(branch), +WriteContext::WriteContext(const std::string& commit_user, bool is_streaming_mode, + bool ignore_num_bucket_check, bool ignore_previous_files, + bool enable_multi_thread_spill, const std::optional& write_id, + const std::vector& write_schema, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::string& temp_directory, + const std::shared_ptr& realtime_context) + : commit_user_(commit_user), is_streaming_mode_(is_streaming_mode), ignore_num_bucket_check_(ignore_num_bucket_check), ignore_previous_files_(ignore_previous_files), @@ -60,13 +47,7 @@ WriteContext::WriteContext( memory_pool_(memory_pool), executor_(executor), temp_directory_(temp_directory), - specific_file_system_(specific_file_system), - fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), - realtime_context_(realtime_context), - options_(options), - format_table_(format_table), - catalog_(catalog), - identifier_(identifier) {} + realtime_context_(realtime_context) {} WriteContext::~WriteContext() = default; @@ -83,26 +64,12 @@ class WriteContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); temp_directory_.clear(); - branch_.reset(); write_schema_.clear(); - fs_scheme_to_identifier_map_.clear(); - specific_file_system_.reset(); realtime_context_.reset(); - catalog_.reset(); - identifier_.reset(); - options_.clear(); } private: - std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one write to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; - /// Unset until `WithBranch()` names one, so that a branch named only by the identifier or - /// the `branch` option is not overruled by the default. - std::optional branch_; std::optional write_id_; bool is_streaming_mode_ = false; bool ignore_num_bucket_check_ = false; @@ -112,44 +79,16 @@ class WriteContextBuilder::Impl { std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); std::string temp_directory_; - std::map fs_scheme_to_identifier_map_; - std::shared_ptr specific_file_system_; std::shared_ptr realtime_context_; - std::shared_ptr catalog_; - std::optional identifier_; - std::map options_; }; -WriteContextBuilder::WriteContextBuilder(const std::string& root_path, - const std::string& commit_user) +WriteContextBuilder::WriteContextBuilder(const std::string& commit_user) : impl_(std::make_unique()) { - impl_->root_path_ = root_path; impl_->commit_user_ = commit_user; } -WriteContextBuilder::WriteContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - WriteContextBuilder::~WriteContextBuilder() = default; -WriteContextBuilder& WriteContextBuilder::AddOption(const std::string& key, - const std::string& value) { - impl_->options_[key] = value; - return *this; -} - -WriteContextBuilder& WriteContextBuilder::SetOptions( - const std::map& opts) { - impl_->options_ = opts; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithStreamingMode(bool is_streaming_mode) { impl_->is_streaming_mode_ = is_streaming_mode; return *this; @@ -186,118 +125,34 @@ WriteContextBuilder& WriteContextBuilder::WithWriteId(int32_t write_id) { return *this; } -WriteContextBuilder& WriteContextBuilder::WithBranch(const std::string& branch) { - impl_->branch_ = branch; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithWriteSchema( const std::vector& write_schema) { impl_->write_schema_ = write_schema; return *this; } -WriteContextBuilder& WriteContextBuilder::WithFileSystemSchemeToIdentifierMap( - const std::map& fs_scheme_to_identifier_map) { - impl_->fs_scheme_to_identifier_map_ = fs_scheme_to_identifier_map; - return *this; -} - WriteContextBuilder& WriteContextBuilder::SetWriteBufferSpillThreadNumber(int32_t thread_number) { impl_->spill_thread_number_ = thread_number; return *this; } -WriteContextBuilder& WriteContextBuilder::WithFileSystem( - const std::shared_ptr& file_system) { - impl_->specific_file_system_ = file_system; - return *this; -} - WriteContextBuilder& WriteContextBuilder::WithRealtimeContext( const std::shared_ptr& realtime_context) { impl_->realtime_context_ = realtime_context; return *this; } -WriteContextBuilder& WriteContextBuilder::WithCatalog(const std::shared_ptr& catalog, - const Identifier& identifier) { - impl_->catalog_ = catalog; - impl_->identifier_.emplace(identifier); - return *this; -} - Result> WriteContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot write with null format table"); - } - if (impl_->format_table_ != nullptr) { - if (impl_->catalog_ != nullptr) { - return Status::Invalid("WithCatalog() requires a native table"); - } - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } - PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); - if (impl_->root_path_.empty()) { - return Status::Invalid("root path is empty"); - } - if (impl_->catalog_ == nullptr && impl_->identifier_) { - return Status::Invalid("cannot write through a null catalog"); - } - if (impl_->identifier_) { - // Before the branch is read out of the identifier: the branch of `tbl$branch_dev$options` - // reads back as `dev`, so a write built for such a name would go to another branch. - PAIMON_RETURN_NOT_OK( - CatalogUtils::CheckNotSystemTable(impl_->identifier_.value(), "write")); - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(impl_->identifier_, impl_->options_, impl_->branch_, "write")); - std::map options = impl_->options_; - std::optional named_branch; - if (impl_->identifier_) { - PAIMON_ASSIGN_OR_RAISE(named_branch, impl_->identifier_->GetBranchName()); - } - if (impl_->branch_ || named_branch || options.count(Options::BRANCH) != 0) { - options[Options::BRANCH] = branch; - } - if (impl_->catalog_ != nullptr && !BranchManager::IsMainBranch(branch)) { - // Refused here rather than at the commit of what is written to the branch. - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_->GetBranchName()); - if (!identifier_branch) { - PAIMON_ASSIGN_OR_RAISE(std::string table_name, impl_->identifier_->GetDataTableName()); - return Status::Invalid(fmt::format( - "a write through a catalog addresses a branch by the table identifier, so name " - "branch '{}' there as '{}$branch_{}'", - branch, table_name, branch)); - } - } bool enable_multi_thread_spill = impl_->spill_thread_number_ > 0; if (enable_multi_thread_spill) { PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow::SetCpuThreadPoolCapacity(impl_->spill_thread_number_)); } auto ctx = std::make_unique( - impl_->root_path_, impl_->commit_user_, impl_->is_streaming_mode_, - impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, - impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, - impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, options, impl_->format_table_, impl_->catalog_, - impl_->identifier_); + impl_->commit_user_, impl_->is_streaming_mode_, impl_->ignore_num_bucket_check_, + impl_->ignore_previous_files_, enable_multi_thread_spill, impl_->write_id_, + impl_->write_schema_, impl_->memory_pool_, impl_->executor_, impl_->temp_directory_, + impl_->realtime_context_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index 1431c154b..de15926e1 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -21,6 +21,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -33,146 +34,23 @@ namespace paimon::test { -TEST(WriteContextTest, TestFailedFinishDoesNotRetainImplicitBranchOption) { - auto catalog = std::make_shared(); - WriteContextBuilder builder("table_root_path", "user"); - builder.WithCatalog(catalog, Identifier("db", "tbl")) - .WithBranch("dev") - .AddOption("custom.option", "value"); - ASSERT_NOK_WITH_MSG(builder.Finish(), "addresses a branch by the table identifier"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.WithBranch("main").Finish()); - ASSERT_EQ("main", context->GetBranch()); - ASSERT_EQ("main", context->GetOptions().at(Options::BRANCH)); - ASSERT_EQ("value", context->GetOptions().at("custom.option")); -} - TEST(WriteContextTest, TestDefaultValue) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); + WriteContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - ASSERT_EQ(ctx->GetRootPath(), "table_root_path"); ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); ASSERT_FALSE(ctx->IsStreamingMode()); ASSERT_FALSE(ctx->IgnoreNumBucketCheck()); ASSERT_FALSE(ctx->IgnorePreviousFiles()); ASSERT_FALSE(ctx->EnableMultiThreadSpill()); ASSERT_EQ(ctx->GetWriteId(), std::nullopt); - ASSERT_EQ(ctx->GetBranch(), "main"); ASSERT_TRUE(ctx->GetWriteSchema().empty()); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); ASSERT_TRUE(ctx->GetTempDirectory().empty()); - ASSERT_TRUE(ctx->GetOptions().empty()); - ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); - ASSERT_FALSE(ctx->GetSpecificFileSystem()); - ASSERT_FALSE(ctx->GetCatalog()); - ASSERT_FALSE(ctx->GetIdentifier()); -} - -TEST(WriteContextTest, TestWithCatalog) { - auto catalog = std::make_shared(); - WriteContextBuilder builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_FALSE(next_ctx->GetCatalog()); - ASSERT_FALSE(next_ctx->GetIdentifier()); - - ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot write through a null catalog"); -} - -TEST(WriteContextTest, TestCatalogAddressesBranchByIdentifier) { - auto catalog = std::make_shared(); - WriteContextBuilder builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .WithBranch("dev") - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ(ctx->GetBranch(), "dev"); - - WriteContextBuilder identifier_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto identifier_ctx, - identifier_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")).Finish()); - ASSERT_EQ(identifier_ctx->GetBranch(), "dev"); - - for (int32_t branch_source = 0; branch_source < 2; ++branch_source) { - SCOPED_TRACE(branch_source); - WriteContextBuilder option_only_builder("table_root_path", "commit_user_1"); - option_only_builder.WithCatalog(catalog, Identifier("db1", "t1")); - if (branch_source == 0) { - option_only_builder.WithBranch("dev"); - } else { - option_only_builder.AddOption(Options::BRANCH, "dev"); - } - ASSERT_NOK_WITH_MSG(option_only_builder.Finish(), - "name branch 'dev' there as 't1$branch_dev'"); - } - - WriteContextBuilder mixed_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(mixed_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_dev")) - .WithBranch("release") - .Finish(), - "but both 'dev' and 'release' were named"); - - WriteContextBuilder main_builder("table_root_path", "commit_user_1"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .WithBranch("") - .AddOption(Options::BRANCH, "main") - .Finish()); - - WriteContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr upper_main_context, - upper_main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_MAIN")).Finish()); - ASSERT_EQ("main", upper_main_context->GetBranch()); - - WriteContextBuilder upper_main_option_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(upper_main_option_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "MAIN") - .Finish(), - "a catalog names branch 'MAIN' as it names the main branch"); - - WriteContextBuilder constructed_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - auto constructed_ctx, - constructed_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev")).Finish()); - ASSERT_EQ(constructed_ctx->GetBranch(), "dev"); - WriteContextBuilder system_table_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG( - system_table_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev$options")).Finish(), - "Cannot 'write' for system table"); -} - -TEST(WriteContextTest, TestBranch) { - WriteContextBuilder option_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto option_ctx, option_builder.AddOption(Options::BRANCH, "rt").Finish()); - ASSERT_EQ("rt", option_ctx->GetBranch()); - - WriteContextBuilder upper_main_builder("table_root_path", "commit_user_1"); - ASSERT_OK_AND_ASSIGN(auto upper_main_ctx, upper_main_builder.WithBranch("MAIN").Finish()); - ASSERT_EQ("MAIN", upper_main_ctx->GetBranch()); - - // An empty branch selects the main branch and stays accepted. - WriteContextBuilder main_builder("table_root_path", "commit_user_1"); - main_builder.WithBranch(""); - ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish()); - ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch()); - WriteContextBuilder empty_builder("table_root_path", "commit_user_1"); - ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH, "dev").Finish(), - "but both 'dev' and 'main' were named"); - - // The branch names a directory under the root path, so a value that is not a single path - // component is rejected when the context is built. - WriteContextBuilder escaping_builder("table_root_path", "commit_user_1"); - escaping_builder.WithBranch("rt/../../../../../outside"); - ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain path separators"); } TEST(WriteContextTest, TestSetContent) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); + WriteContextBuilder builder("commit_user_1"); auto memory_pool = GetDefaultPool(); std::shared_ptr executor = CreateDefaultExecutor(); @@ -181,20 +59,15 @@ TEST(WriteContextTest, TestSetContent) { std::map fs_scheme_to_identifier_map = {{"file", "local"}, {"oss", "jindo"}}; - ASSERT_OK_AND_ASSIGN(auto ctx, - builder.WithStreamingMode(true) - .WithIgnoreNumBucketCheck(true) - .WithIgnorePreviousFiles(true) - .WithMemoryPool(memory_pool) - .WithExecutor(executor) - .WithTempDirectory("/tmp/with-all") - .WithWriteId(123) - .WithBranch("test_branch") - .WithWriteSchema(write_schema) - .WithFileSystemSchemeToIdentifierMap(fs_scheme_to_identifier_map) - .WithFileSystem(file_system) - .AddOption("key", "value") - .Finish()); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithStreamingMode(true) + .WithIgnoreNumBucketCheck(true) + .WithIgnorePreviousFiles(true) + .WithMemoryPool(memory_pool) + .WithExecutor(executor) + .WithTempDirectory("/tmp/with-all") + .WithWriteId(123) + .WithWriteSchema(write_schema) + .Finish()); ASSERT_TRUE(ctx->IsStreamingMode()); ASSERT_TRUE(ctx->IgnoreNumBucketCheck()); @@ -204,28 +77,11 @@ TEST(WriteContextTest, TestSetContent) { ASSERT_EQ(ctx->GetExecutor(), executor); ASSERT_EQ(ctx->GetTempDirectory(), "/tmp/with-all"); ASSERT_EQ(ctx->GetWriteId(), 123); - ASSERT_EQ(ctx->GetBranch(), "test_branch"); ASSERT_EQ(ctx->GetWriteSchema(), write_schema); - ASSERT_EQ(ctx->GetFileSystemSchemeToIdentifierMap(), fs_scheme_to_identifier_map); - ASSERT_EQ(ctx->GetSpecificFileSystem(), file_system); - std::map expected_options = {{"key", "value"}, - {Options::BRANCH, "test_branch"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); -} - -TEST(WriteContextTest, TestSetOptionsOverridesAddedOptions) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); - builder.AddOption("old", "value"); - builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); - - ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); - - std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; - ASSERT_EQ(expected_options, ctx->GetOptions()); } TEST(WriteContextTest, TestSetWriteBufferSpillThreadNumber) { - WriteContextBuilder builder("table_root_path", "commit_user_1"); + WriteContextBuilder builder("commit_user_1"); builder.SetWriteBufferSpillThreadNumber(2); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); diff --git a/src/paimon/core/snapshot_file_scan_test.cpp b/src/paimon/core/snapshot_file_scan_test.cpp index 3f8bba11b..e9facf193 100644 --- a/src/paimon/core/snapshot_file_scan_test.cpp +++ b/src/paimon/core/snapshot_file_scan_test.cpp @@ -32,6 +32,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/utils/path_util.h" @@ -465,14 +466,16 @@ TEST(SnapshotFileScanTest, TestCommittedRealtimeOffsetFiles) { const std::string table_path = dir->Str() + "/foo.db/bar"; ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder write_builder(table_path, "test"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.AddOption(Options::REALTIME_ENABLED, "true") - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context) - .Finish()); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("test"); + write_builder_options[Options::REALTIME_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + write_builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto input_type = arrow::struct_({arrow::field("_REALTIME_OFFSET", arrow::int64(), false), schema->field(0), schema->field(1)}); ASSERT_OK_AND_ASSIGN( @@ -481,11 +484,15 @@ TEST(SnapshotFileScanTest, TestCommittedRealtimeOffsetFiles) { ASSERT_OK(writer->Write(std::move(batch))); ASSERT_OK_AND_ASSIGN(auto progress, writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); ASSERT_FALSE(progress.empty()); - CommitContextBuilder commit_builder(table_path, "test"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.AddOption(Options::REALTIME_ENABLED, "true").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test"); + commit_builder_options[Options::REALTIME_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, commit->CommitWithProgress(progress, 0, std::nullopt)); ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id)); diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 0dbc1bf4f..49323b386 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -153,41 +153,41 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& return Status::OK(); } -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; +bool IsCatalogCommit(const std::shared_ptr& catalog) { + return AsVersionManaged(catalog) != nullptr; } Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, + const CommitContext& ctx, const std::shared_ptr& catalog, + const std::optional& table_id, const std::optional& catalog_identifier, const CoreOptions& options, const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { + if (IsCatalogCommit(catalog)) { if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr lock, - AsVersionManaged(ctx.GetCatalog()) - ->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(), - ctx.GetCommitUser(), options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr lock, + AsVersionManaged(catalog)->CreateCommitLock(catalog_identifier.value(), table_id, + ctx.GetCommitUser(), options)); // Reuse the caller's UUID so table recreation cannot redirect this commit. return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(), std::move(lock))); + catalog, catalog_identifier.value(), table_id, std::move(lock))); } if (ctx.UseRESTCatalogCommit()) { if (options.RestCommitAcquireLock()) { - return Status::Invalid(fmt::format( - "'{}' needs a commit through a REST catalog with WithCatalog(); a commit table " - "request built for the caller to send cannot hold a commit lease", - Options::REST_COMMIT_ACQUIRE_LOCK)); + return Status::Invalid( + fmt::format("'{}' needs a table loaded through a REST catalog; a commit table " + "request built for the caller to send cannot hold a commit lease", + Options::REST_COMMIT_ACQUIRE_LOCK)); } - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); + return std::shared_ptr(std::make_shared(table_id)); } // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { + if (table_id && catalog == nullptr) { return Status::Invalid( "a commit which publishes the snapshot by writing it to the table directory sends no " "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + "catalog or ask for the request with UseRESTCatalogCommit()"); } return std::shared_ptr( std::make_shared(options.GetFileSystem(), snapshot_manager)); @@ -211,7 +211,9 @@ Result> FileStoreTable::Create( const std::shared_ptr& specified_file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, - const std::map& fs_scheme_map) { + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, + const std::map& dynamic_options) { if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema of {}", identifier.GetFullName())); @@ -233,13 +235,15 @@ Result> FileStoreTable::Create( effective_schema = std::shared_ptr(std::move(normalized_schema)); } return CreateWithOptions(specified_file_system, root_path, effective_schema, identifier, uuid, - options, fs_scheme_map); + options, fs_scheme_map, catalog, dynamic_options); } Result> FileStoreTable::Create( std::unique_ptr store, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::map& fs_scheme_map, + const std::shared_ptr& catalog, + const std::map& dynamic_options, const std::shared_ptr& schema_manager) { if (!store || !schema) { return Status::Invalid("managed table requires a store and schema"); @@ -250,7 +254,7 @@ Result> FileStoreTable::Create( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr table, CreateWithOptions(store->GetFileSystem(), store->GetRootPath(), schema, identifier, uuid, - store->GetOptions().ToMap(), fs_scheme_map)); + store->GetOptions().ToMap(), fs_scheme_map, catalog, dynamic_options)); if (schema_manager) { table->schema_manager_ = schema_manager; } @@ -261,7 +265,9 @@ Result> FileStoreTable::CreateWithOptions( const std::shared_ptr& specified_file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::map& options, - const std::map& fs_scheme_map) { + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, + const std::map& dynamic_options) { // Loading metadata must not require the data-format plugins used by operators. std::shared_ptr file_system = specified_file_system; if (!file_system) { @@ -279,10 +285,12 @@ Result> FileStoreTable::CreateWithOptions( identifier, uuid, options); } table->fs_scheme_map_ = fs_scheme_map; + table->catalog_ = catalog; + table->dynamic_options_ = dynamic_options; return table; } -Result FileStoreTable::ResolveOptions( +Result> FileStoreTable::ResolveOptionMap( const std::map& overrides) const { auto options = options_; for (const auto& [key, value] : overrides) { @@ -291,16 +299,45 @@ Result FileStoreTable::ResolveOptions( auto branch = overrides.find(Options::BRANCH); if (branch != overrides.end() && BranchManager::NormalizeBranch(branch->second) != CurrentBranch()) { - return Status::Invalid("context branch contradicts the loaded table"); + return Status::Invalid("copy branch contradicts the loaded table"); } PAIMON_ASSIGN_OR_RAISE(TableType type, TableTypeDefine::FromOptions(options)); PAIMON_ASSIGN_OR_RAISE(TableType bound_type, TableTypeDefine::FromOptions(options_)); if (type != bound_type) { - return Status::Invalid("context type contradicts the loaded table"); + return Status::Invalid("copy type contradicts the loaded table"); } + return options; +} + +Result FileStoreTable::ResolveOptions( + const std::map& overrides) const { + using OptionsMap = std::map; + PAIMON_ASSIGN_OR_RAISE(OptionsMap options, ResolveOptionMap(overrides)); return CoreOptions::FromMap(options, file_system_, fs_scheme_map_); } +Result> FileStoreTable::Copy( + const std::map& dynamic_options) const { + using OptionsMap = std::map; + PAIMON_ASSIGN_OR_RAISE(OptionsMap options, ResolveOptionMap(dynamic_options)); + PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] bool query_auth, + SchemaValidation::QueryAuthReadEnabled(options_, dynamic_options)); + auto explicit_options = dynamic_options_; + for (const auto& [key, value] : dynamic_options) { + explicit_options[key] = value; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + checked_pointer_cast(schema_)->Copy(options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + CreateWithOptions(file_system_, root_path_, schema, identifier_, CatalogUuid().value_or(""), + options, fs_scheme_map_, catalog_, explicit_options)); + table->schema_manager_ = schema_manager_; + table->supplied_main_schema_ = supplied_main_schema_; + table->fallback_file_system_ = fallback_file_system_; + return std::shared_ptr
(std::move(table)); +} + Status FileStoreTable::RestoreRealtimeCommittedProgress( const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options) { @@ -319,63 +356,6 @@ Status FileStoreTable::RestoreRealtimeCommittedProgress( return Status::OK(); } -Status FileStoreTable::CheckContextNamesThisTable( - const std::string& kind, const std::string& context_root_path, - const std::optional& context_identifier, - const std::optional& context_table_id) const { - if (!root_path_.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string context_path, - PathUtil::NormalizePath(context_root_path)); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, PathUtil::NormalizePath(root_path_)); - if (context_path != table_path) { - return Status::Invalid(fmt::format( - "the {} context names root path '{}' but this table serves '{}': the store below " - "reads and writes this table's files, so a context naming another path would " - "leave this table's metadata there and publish it as the other's", - kind, context_root_path, root_path_)); - } - } - if (context_identifier) { - PAIMON_ASSIGN_OR_RAISE(std::optional context_branch, - context_identifier.value().GetBranchName()); - if (context_branch) { - PAIMON_ASSIGN_OR_RAISE(context_branch, - context_identifier.value().GetBranchNameOrDefault()); - } - if (context_branch && context_branch.value() != CurrentBranch()) { - return Status::Invalid(fmt::format( - "the {} context names branch '{}' of '{}' but this table serves branch '{}', " - "which is the one a {} of this table writes and publishes", - kind, context_branch.value(), context_identifier.value().ToString(), - CurrentBranch(), kind)); - } - PAIMON_ASSIGN_OR_RAISE( - Identifier context_table, - CatalogUtils::BranchIdentifier(context_identifier.value(), CurrentBranch())); - PAIMON_ASSIGN_OR_RAISE(Identifier served_table, - CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); - const bool names_this_table = - (served_table.GetDatabaseName() == Identifier::kUnknownDatabase || - context_table.GetDatabaseName() == served_table.GetDatabaseName()) && - context_table.GetObjectName() == served_table.GetObjectName(); - if (!names_this_table) { - return Status::Invalid(fmt::format( - "the {} context names table '{}' but this table is '{}': the store below serves " - "this table's files, so a context naming another table would write this table's " - "metadata and publish it as the other's", - kind, context_identifier.value().ToString(), identifier_.ToString())); - } - } - if (context_table_id && context_table_id != CatalogUuid()) { - return Status::Invalid(fmt::format( - "the {} context carries table id '{}' but this table's is '{}': the id tells a " - "recreated table from the one the changes were prepared for, so one this table does " - "not carry would publish this table's metadata as the recreation's", - kind, context_table_id.value(), CatalogUuid().value_or("none"))); - } - return Status::OK(); -} - std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, const std::string& root_path) { scan->WithTablePath(root_path); @@ -384,30 +364,23 @@ std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr> FileStoreTable::NewWrite( std::unique_ptr ctx) const { + const auto& dynamic_options = dynamic_options_; PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); - PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write", ctx->GetRootPath(), - ctx->GetIdentifier(), - /*context_table_id=*/std::nullopt)); - if (AsVersionManaged(ctx->GetCatalog()) != nullptr) { + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(catalog_)); + if (AsVersionManaged(catalog_)) { PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier identifier, CatalogUtils::BranchIdentifier( - *ctx->GetIdentifier(), CurrentBranch())); + if (catalog_) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); catalog_identifier.emplace(std::move(identifier)); } if (ctx->GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); } std::shared_ptr schema = checked_pointer_cast(schema_); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(ctx->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(dynamic_options)); FileStore file_store(root_path_, options); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); @@ -425,7 +398,7 @@ Result> FileStoreTable::NewWrite( options.GetWriteFileFormat(/*level=*/0)->Identifier(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); + file_store.NewSnapshotManager(catalog_, catalog_identifier)); std::shared_ptr io_manager; const auto& io_temp_dir = ctx->GetTempDirectory(); if (!io_temp_dir.empty()) { @@ -452,6 +425,7 @@ Result> FileStoreTable::NewScan( Result> FileStoreTable::NewScan( const std::shared_ptr& context, bool query_auth_applied) const { + const auto& dynamic_options = dynamic_options_; PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); @@ -460,24 +434,19 @@ Result> FileStoreTable::NewScan( std::map read_options; PAIMON_ASSIGN_OR_RAISE( read_options, CatalogUtils::ResolveQueryAuthReadOptions( - context->GetCatalog(), context->GetIdentifier(), context->GetOptions(), - CurrentBranch(), file_system_, root_path_, true, "scan")); + catalog_, identifier_, dynamic_options, CurrentBranch(), file_system_, + root_path_, true, "scan")); PAIMON_ASSIGN_OR_RAISE(bool query_auth, SchemaValidation::QueryAuthReadEnabled( table_schema->Options(), read_options)); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, ResolveOptions(read_options)); - if (query_auth) { - // The authorization scope and the bound data source must name the same table. - PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("scan", context->GetPath(), - context->GetIdentifier(), std::nullopt)); - } if (query_auth && !query_auth_applied) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr table, FileStoreTable::Create(std::make_unique(root_path_, core_options), table_schema, identifier_, CatalogUuid().value_or(""), - fs_scheme_map_)); + fs_scheme_map_, catalog_, dynamic_options_, schema_manager_)); return QueryAuthTableScan::Create( - context, table_schema, file_system_, CurrentBranch(), + context, table_schema, file_system_, CurrentBranch(), catalog_, identifier_, root_path_, [table](const std::shared_ptr& inner_context) { return table->NewScan(inner_context, true); }); @@ -602,6 +571,7 @@ Result> FileStoreTable::NewRead( Result> FileStoreTable::NewRead( const std::shared_ptr& read_context, bool query_auth_applied) const { + const auto& dynamic_options = dynamic_options_; PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); if (read_context->GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch())); @@ -609,37 +579,30 @@ Result> FileStoreTable::NewRead( std::shared_ptr loaded_schema = checked_pointer_cast(schema_); std::map read_options; PAIMON_ASSIGN_OR_RAISE( - read_options, - CatalogUtils::ResolveQueryAuthReadOptions( - read_context->GetCatalog(), read_context->GetIdentifier(), read_context->GetOptions(), - CurrentBranch(), file_system_, root_path_, true, "read")); + read_options, CatalogUtils::ResolveQueryAuthReadOptions( + catalog_, identifier_, dynamic_options, CurrentBranch(), file_system_, + root_path_, true, "read")); PAIMON_ASSIGN_OR_RAISE(bool query_auth, SchemaValidation::QueryAuthReadEnabled( loaded_schema->Options(), read_options)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(read_options)); - if (query_auth) { - // The authorization scope and the bound data source must name the same table. - PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable( - "read", read_context->GetPath(), read_context->GetIdentifier(), std::nullopt)); - } if (query_auth && !query_auth_applied) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr table, FileStoreTable::Create(std::make_unique(root_path_, options), loaded_schema, - identifier_, CatalogUuid().value_or(""), fs_scheme_map_)); + identifier_, CatalogUuid().value_or(""), fs_scheme_map_, + catalog_, dynamic_options_, schema_manager_)); return QueryAuthTableRead::Create( - read_context, loaded_schema, GetArrowPool(read_context->GetMemoryPool()), + read_context, loaded_schema, root_path_, CurrentBranch(), + GetArrowPool(read_context->GetMemoryPool()), [table](std::unique_ptr inner_context) { return table->NewRead(std::shared_ptr(std::move(inner_context)), true); }); } - auto fs_scheme_map = fs_scheme_map_; - for (const auto& [scheme, identifier] : read_context->GetFileSystemSchemeToIdentifierMap()) { - fs_scheme_map[scheme] = identifier; - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::CreateForTable(read_context, loaded_schema, std::move(options), - root_path_, fs_scheme_map)); + const auto& explicit_options = dynamic_options_; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::CreateForTable( + read_context, loaded_schema, std::move(options), root_path_, + fs_scheme_map_, catalog_, identifier_, explicit_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -658,15 +621,15 @@ Result> FileStoreTable::NewRead( "own authorized splits and schema validation."); } std::map fallback_read_options; - PAIMON_ASSIGN_OR_RAISE( - fallback_read_options, - CatalogUtils::ResolveQueryAuthReadOptions( - read_context->GetCatalog(), read_context->GetIdentifier(), read_context->GetOptions(), - scan_fallback_branch.value(), file_system_, root_path_, true, "read")); + PAIMON_ASSIGN_OR_RAISE(fallback_read_options, + CatalogUtils::ResolveQueryAuthReadOptions( + catalog_, identifier_, dynamic_options, scan_fallback_branch.value(), + file_system_, root_path_, true, "read")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, InternalReadContext::CreateForFallbackBranch( read_context, /*branch=*/scan_fallback_branch.value(), file_system_, - root_path_, fs_scheme_map, loaded_from_read_context_)); + root_path_, fs_scheme_map_, catalog_, identifier_, explicit_options, + supplied_main_schema_, fallback_file_system_)); PAIMON_ASSIGN_OR_RAISE( bool fallback_query_auth, @@ -688,29 +651,28 @@ Result> FileStoreTable::NewRead( Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { + const auto& dynamic_options = dynamic_options_; PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("commit", ctx->GetRootPath(), - ctx->GetIdentifier(), ctx->GetTableId())); - if (ctx->GetCatalog() != nullptr || ctx->UseRESTCatalogCommit()) { - PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(catalog_)); + if (catalog_ && ctx->UseRESTCatalogCommit()) { + return Status::Invalid( + "a commit cannot both use a catalog and build an external REST request"); } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); + if (catalog_ || ctx->UseRESTCatalogCommit()) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); } std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier identifier, CatalogUtils::BranchIdentifier( - *ctx->GetIdentifier(), CurrentBranch())); + if (catalog_) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); catalog_identifier.emplace(std::move(identifier)); } FileStoreCommitImpl::SchemaIdLoader schema_id_loader; - if (ctx->GetCatalog() != nullptr) { + if (catalog_ != nullptr) { if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - std::shared_ptr catalog = ctx->GetCatalog(); + std::shared_ptr catalog = catalog_; Identifier identifier = catalog_identifier.value(); schema_id_loader = [catalog, identifier]() -> Result { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, @@ -727,15 +689,14 @@ Result> FileStoreTable::NewCommit( std::shared_ptr schema = checked_pointer_cast(schema_); std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(ctx->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, ResolveOptions(dynamic_options)); FileStore file_store(root_path_, options); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); - if (is_object_store && !ctx->UseRESTCatalogCommit() && - AsVersionManaged(ctx->GetCatalog()) == nullptr && + if (is_object_store && !ctx->UseRESTCatalogCommit() && AsVersionManaged(catalog_) == nullptr && options.ToMap().count(kObjectStoreCommitInteTestOption) == 0) { return Status::NotImplemented( "commit operation does not support object store file system for now"); @@ -749,7 +710,7 @@ Result> FileStoreTable::NewCommit( options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, - file_store.NewSnapshotManager(ctx->GetCatalog(), catalog_identifier)); + file_store.NewSnapshotManager(catalog_, catalog_identifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, file_store.NewManifestList(path_factory, ctx->GetMemoryPool())); @@ -773,7 +734,8 @@ Result> FileStoreTable::NewCommit( ctx->GetExecutor(), ctx->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options, snapshot_manager)); + NewSnapshotCommit(*ctx, catalog_, CatalogUuid(), catalog_identifier, + options, snapshot_manager)); return std::make_unique( ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index e9e34adf6..97f0168a6 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -36,6 +36,7 @@ namespace paimon { +class Catalog; class InternalReadContext; class IOManager; class ManifestFile; @@ -56,7 +57,9 @@ class FileStoreTable : public Table { const std::shared_ptr& specified_file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, - const std::map& fs_scheme_map = {}); + const std::map& fs_scheme_map = {}, + const std::shared_ptr& catalog = nullptr, + const std::map& dynamic_options = {}); /// Binds a resolved store without replacing the loaded schema's options. /// @param schema_manager Manager used to load this store's schema, preserving its cache. @@ -64,8 +67,14 @@ class FileStoreTable : public Table { std::unique_ptr store, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::map& fs_scheme_map = {}, + const std::shared_ptr& catalog = nullptr, + const std::map& dynamic_options = {}, const std::shared_ptr& schema_manager = nullptr); + const std::map& GetFileSystemSchemeToIdentifierMap() const { + return fs_scheme_map_; + } + std::string CurrentBranch() const { auto branch = options_.find(Options::BRANCH); return branch == options_.end() ? BranchManager::DEFAULT_MAIN_BRANCH @@ -88,6 +97,9 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } + Result> Copy( + const std::map& dynamic_options) const override; + Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; @@ -100,8 +112,10 @@ class FileStoreTable : public Table { private: friend class TableFactory; - // Only the legacy static factory may use its context to load the fallback branch. - bool loaded_from_read_context_ = false; + // A static load may supply the main schema while reading a different branch. + std::optional supplied_main_schema_; + // Engaged for static loads; nullptr means resolve storage from the fallback schema options. + std::optional> fallback_file_system_; Result> NewScan(const std::shared_ptr& context, bool query_auth_applied) const; @@ -113,17 +127,14 @@ class FileStoreTable : public Table { const std::shared_ptr& specified_file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::map& options, - const std::map& fs_scheme_map); + const std::map& fs_scheme_map, + const std::shared_ptr& catalog, + const std::map& dynamic_options); - Result ResolveOptions(const std::map& options) const; - /// Rejects paths, identifiers or UUIDs that contradict this table's identity. - /// Missing context fields are allowed. - /// @param kind Operation name used in errors. - /// @param context_table_id Catalog commit UUID; nullopt for writes. - Status CheckContextNamesThisTable(const std::string& kind, const std::string& context_root_path, - const std::optional& context_identifier, - const std::optional& context_table_id) const; + Result> ResolveOptionMap( + const std::map& overrides) const; + Result ResolveOptions(const std::map& options) const; /// Creates a reader for the internal context's schema and primary keys. Result> CreateTableRead( const std::shared_ptr& internal_context, @@ -173,6 +184,8 @@ class FileStoreTable : public Table { const std::map options_; std::shared_ptr schema_manager_; + std::shared_ptr catalog_; + std::map dynamic_options_; /// URI routing supplied when this table was loaded. std::map fs_scheme_map_; diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 46647d11e..0bd1f033c 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -223,11 +223,6 @@ Result> FormatTable::GetArrowSchema() const { namespace { Result> NewFormatTableWrite( const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } if (ctx.IsStreamingMode()) { return Status::NotImplemented( "a format table has no snapshots, so there is nothing a streaming write could " @@ -316,15 +311,6 @@ Result> NewFormatTableCommit( "a format table commits by renaming files into place, not by sending a snapshot " "to a rest catalog"); } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } if (ctx.AppendCommitCheckConflict()) { return Status::NotImplemented( "a format table has no manifests to check a concurrent commit against"); @@ -334,66 +320,35 @@ Result> NewFormatTableCommit( return std::unique_ptr(std::move(format_commit)); } } // namespace -Status FormatTable::CheckWriteContext(const std::string& path, - const std::shared_ptr& context_table) const { - PAIMON_ASSIGN_OR_RAISE(std::string context_path, PathUtil::NormalizePath(path)); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, PathUtil::NormalizePath(Location())); - if (context_path != table_path) { - return Status::Invalid("write/commit context path contradicts the loaded format table"); - } - if (context_table) { - // Equal URIs do not establish equal endpoints or credentials. A typed context must - // reuse this storage binding (including copies of this table), not a separately loaded one. - if (file_system_ != context_table->GetFileSystem()) { - return Status::Invalid( - "write/commit context filesystem contradicts the loaded format table"); - } - const Identifier& other = context_table->identifier_; - const bool database_matches = - identifier_.GetDatabaseName() == Identifier::kUnknownDatabase || - other.GetDatabaseName() == Identifier::kUnknownDatabase || - identifier_.GetDatabaseName() == other.GetDatabaseName(); - PAIMON_ASSIGN_OR_RAISE(std::string table_name, identifier_.GetDataTableName()); - PAIMON_ASSIGN_OR_RAISE(std::string other_name, other.GetDataTableName()); - PAIMON_ASSIGN_OR_RAISE(std::string table_branch, identifier_.GetBranchNameOrDefault()); - PAIMON_ASSIGN_OR_RAISE(std::string other_branch, other.GetBranchNameOrDefault()); - if (!database_matches || table_name != other_name || table_branch != other_branch) { - return Status::Invalid( - "write/commit context identity contradicts the loaded format table"); - } - } - return Status::OK(); -} - -Result> FormatTable::ForContext( +Result> FormatTable::Copy( const std::map& options) const { - return Copy(std::make_shared(*this), options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + Copy(std::make_shared(*this), options)); + return std::shared_ptr
(std::move(table)); } Result> FormatTable::NewScan(std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "scan")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + auto table = std::make_shared(*this); std::shared_ptr context = std::move(ctx); return NewFormatTableScan(table, context); } Result> FormatTable::NewRead(std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + auto table = std::make_shared(*this); std::shared_ptr context = std::move(ctx); return NewFormatTableRead(table, context); } Result> FormatTable::NewWrite( std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); - PAIMON_RETURN_NOT_OK(CheckWriteContext(ctx->GetRootPath(), ctx->GetFormatTable())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + auto table = std::make_shared(*this); std::shared_ptr context = std::move(ctx); return NewFormatTableWrite(table, *context); } Result> FormatTable::NewCommit( std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - PAIMON_RETURN_NOT_OK(CheckWriteContext(ctx->GetRootPath(), ctx->GetFormatTable())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForContext(ctx->GetOptions())); + auto table = std::make_shared(*this); std::shared_ptr context = std::move(ctx); return NewFormatTableCommit(table, *context); } diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 44b23fb9c..bba59fcb6 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -34,6 +34,8 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/table.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/table/special_fields.h" @@ -656,8 +658,9 @@ TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { for (int32_t entrypoint : {0, 1, 2}) { SCOPED_TRACE(vector_search); SCOPED_TRACE(entrypoint); - ReadContextBuilder builder = - entrypoint == 2 ? ReadContextBuilder(table) : ReadContextBuilder(dir->Str()); + ReadContextBuilder builder; + TableLoadOptions context_load_options{dir->Str()}; + std::map context_options; builder.EnablePreciseFileIndexEvaluate(true); if (vector_search) { builder.SetVectorSearch(std::make_shared( @@ -668,16 +671,26 @@ TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { std::make_shared("name", R"({"match":{"query":"paimon"}})", 3)); } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - auto read = entrypoint == 0 ? loaded_table->NewRead(std::move(context)) - : TableRead::Create(std::move(context)); + auto read = entrypoint != 1 ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + loaded_table->Copy(context_options)); + return configured->NewRead(std::move(context)); + })() + : TableRead::Create(context_load_options, + std::move(context), context_options); ASSERT_NOK_WITH_MSG(read, "Precise file-index evaluation is not supported for format tables"); ASSERT_TRUE(read.status().IsNotImplemented()); } } - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(dir->Str()).Finish()); - ASSERT_OK(loaded_table->NewRead(std::move(context))); + std::map context_builder_options; + ReadContextBuilder context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder.Finish()); + ASSERT_OK(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + loaded_table->Copy(context_builder_options)); + return configured->NewRead(std::move(context)); + })()); } TEST(FormatTableTest, TestFileCompressionComesFromCoreOptions) { @@ -808,26 +821,44 @@ TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { const std::map refused = { {Options::METASTORE_PARTITIONED_TABLE, "true"}}; - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions(refused); + TableLoadOptions scan_builder_load_options{dir->Str()}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = refused; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG( + TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options), + "metastore.partitioned-table"); + + TableLoadOptions read_builder_load_options{dir->Str()}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = refused; - ReadContextBuilder read_builder(dir->Str()); - read_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_builder_load_options, std::move(read_context), read_builder_options), + "metastore.partitioned-table"); + + TableLoadOptions write_builder_load_options{dir->Str()}; + std::map write_builder_options; + WriteContextBuilder write_builder("test-user"); + write_builder_options = refused; - WriteContextBuilder write_builder(dir->Str(), "test-user"); - write_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options), "metastore.partitioned-table"); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); - commit_builder.SetOptions(refused); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); + commit_builder_options = refused; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(commit_builder_load_options, + std::move(commit_context), commit_builder_options), "metastore.partitioned-table"); } @@ -855,17 +886,25 @@ TEST(FormatTableTest, TestQueryAuthInTheSchemaCannotBeDisabledAtTheCall) { ASSERT_TRUE(created.IsInvalid()) << created.ToString(); ASSERT_NOK_WITH_MSG(created, refused); - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions(disabling); + TableLoadOptions scan_builder_load{dir->Str()}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = disabling; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - Status scanned = TableScan::Create(std::move(scan_context)).status(); + Status scanned = + TableScan::Create(scan_builder_load, std::move(scan_context), scan_builder_options) + .status(); ASSERT_TRUE(scanned.IsInvalid()) << scanned.ToString(); ASSERT_NOK_WITH_MSG(scanned, refused); - ReadContextBuilder read_builder(dir->Str()); - read_builder.SetOptions(disabling); + TableLoadOptions read_builder_load{dir->Str()}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = disabling; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - Status read = TableRead::Create(std::move(read_context)).status(); + Status read = + TableRead::Create(read_builder_load, std::move(read_context), read_builder_options) + .status(); ASSERT_TRUE(read.IsInvalid()) << read.ToString(); ASSERT_NOK_WITH_MSG(read, refused); } @@ -1124,6 +1163,96 @@ TEST(FormatTableTest, TestExternalLocationHasNoReservedDirectories) { ASSERT_EQ(rows, (std::vector{"1|alice|schema"})); } +TEST(FormatTableTest, TestCatalogFactoriesRetainExternalLocationWithExplicitPath) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(0, MakeSchema(), {"dt"}, {}, + {{Options::TYPE, "format-table"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE, "true"}})); + for (bool explicit_path : {false, true}) { + SCOPED_TRACE(explicit_path); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr external, + FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), + checked_pointer_cast(schema), + /*location_carries_paimon_metadata=*/false, {})); + auto catalog = std::make_shared(); + catalog->SetTable(external); + TableLoadOptions load; + load.catalog = catalog; + load.identifier.emplace("db", "tbl"); + if (explicit_path) { + load.path = dir->Str() + "/"; + } + int32_t id = 0; + for (const std::string partition : {"schema", "branch"}) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + WriteContextBuilder("writer").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + FileStoreWrite::Create(load, std::move(write_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch({++id}, {partition}, partition, {{"dt", partition}})); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + writer->PrepareCommit()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + CommitContextBuilder("writer").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + FileStoreCommit::Create(load, std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(writer->Close()); + } + std::vector files; + ASSERT_OK(dir->GetFileSystem()->ListDir(dir->Str() + "/schema", &files)); + bool renamed = false; + for (const auto& file : files) { + if (!file.IsDir() && StringUtils::EndsWith(file.GetPath(), ".parquet")) { + ASSERT_FALSE(renamed); + ASSERT_OK(dir->GetFileSystem()->Rename(file.GetPath(), + dir->Str() + "/schema/schema-0.parquet")); + renamed = true; + } + } + ASSERT_TRUE(renamed); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(load, std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 2u); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + TableRead::Create(load, std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader(plan->Splits())); + std::vector partitions; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, ImportBatch(batch)); + auto values = checked_pointer_cast(rows->GetColumnByName("dt")); + for (int64_t row = 0; row < rows->num_rows(); ++row) { + partitions.push_back(values->GetString(row)); + } + } + reader->Close(); + std::sort(partitions.begin(), partitions.end()); + ASSERT_EQ(partitions, (std::vector{"branch", "schema"})); + ASSERT_TRUE(catalog->CommitCalls().empty()); + load.path = dir->Str() + "/other"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr mismatch_context, + ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(load, std::move(mismatch_context)), + "does not match the catalog table location"); + } +} + TEST(FormatTableTest, TestFileSuffixIncludesCompressionWhenAsked) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -2748,10 +2877,13 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - WriteContextBuilder write_builder(dir->Str(), "test-user"); + TableLoadOptions write_builder_load_options{dir->Str()}; + std::map write_builder_options; + WriteContextBuilder write_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_NOK_WITH_MSG(write->GetBaseSnapshotId({{"dt", "20240101"}}, 0), "format tables do not have base snapshots"); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, @@ -2767,67 +2899,105 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { "cannot be compacted"); // A write id prefixes a postpone-bucket writer's files; a format table has no buckets. - WriteContextBuilder write_id_builder(dir->Str(), "test-user"); + TableLoadOptions write_id_builder_load_options{dir->Str()}; + std::map write_id_builder_options; + WriteContextBuilder write_id_builder("test-user"); write_id_builder.WithWriteId(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_context, write_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id_context)), - "a write id would name nothing"); + ASSERT_NOK_WITH_MSG( + FileStoreWrite::Create(write_id_builder_load_options, std::move(write_id_context), + write_id_builder_options), + "a write id would name nothing"); - CommitContextBuilder empty_commit_builder(dir->Str(), "test-user"); + TableLoadOptions empty_commit_builder_load_options{dir->Str()}; + std::map empty_commit_builder_options; + CommitContextBuilder empty_commit_builder("test-user"); empty_commit_builder.IgnoreEmptyCommit(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_commit_context, empty_commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(empty_commit_context)), - "cannot record an empty commit"); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(empty_commit_builder_load_options, std::move(empty_commit_context), + empty_commit_builder_options), + "cannot record an empty commit"); - CommitContextBuilder rest_builder(dir->Str(), "test-user"); + TableLoadOptions rest_builder_load_options{dir->Str()}; + std::map rest_builder_options; + CommitContextBuilder rest_builder("test-user"); rest_builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, rest_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(rest_context)), "rest catalog"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(rest_builder_load_options, std::move(rest_context), + rest_builder_options), + "rest catalog"); + + TableLoadOptions table_id_builder_load_options{dir->Str()}; + std::map table_id_builder_options; + CommitContextBuilder table_id_builder("test-user"); + table_id_builder_load_options.table_id = "table-uuid"; - CommitContextBuilder table_id_builder(dir->Str(), "test-user"); - table_id_builder.WithTableId("table-uuid"); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_id_context, table_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(table_id_context)), - "a format table sends no commit table request"); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(table_id_builder_load_options, std::move(table_id_context), + table_id_builder_options), + "a table id requires a catalog or an external REST commit request"); auto format_catalog = std::make_shared(); - format_catalog->SetTableSchema(table->GetSchema()); - CommitContextBuilder catalog_builder(dir->Str(), "test-user"); - catalog_builder.WithCatalog(format_catalog, Identifier("db", "fmt")); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_context)), - "keeps no snapshot for a catalog to take"); + format_catalog->SetTable(table); + TableLoadOptions catalog_builder_load_options{dir->Str()}; + std::map catalog_builder_options; + CommitContextBuilder catalog_builder("test-user"); + catalog_builder_load_options.catalog = format_catalog; + catalog_builder_load_options.identifier.emplace(Identifier("db", "fmt")); - CommitContextBuilder conflict_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr catalog_commit, + FileStoreCommit::Create(catalog_builder_load_options, std::move(catalog_context), + catalog_builder_options)); + ASSERT_TRUE(format_catalog->CommitCalls().empty()); + + TableLoadOptions conflict_builder_load_options{dir->Str()}; + std::map conflict_builder_options; + CommitContextBuilder conflict_builder("test-user"); conflict_builder.AppendCommitCheckConflict(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr conflict_context, conflict_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(conflict_context)), - "no manifests to check a concurrent commit against"); + ASSERT_NOK_WITH_MSG( + FileStoreCommit::Create(conflict_builder_load_options, std::move(conflict_context), + conflict_builder_options), + "no manifests to check a concurrent commit against"); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK(commit->Commit(messages)); // The same for the snapshot half of the commit interface. ASSERT_NOK_WITH_MSG(commit->Expire(), "no snapshots to expire"); // Closing after a prepared commit must not take back what the commit just published. ASSERT_OK(write->Close()); - ScanContextBuilder scan_builder(dir->Str()); + TableLoadOptions scan_builder_load_options{dir->Str()}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1u); - ReadContextBuilder read_builder(dir->Str()); + TableLoadOptions read_builder_load_options{dir->Str()}; + std::map read_builder_options; + ReadContextBuilder read_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); ASSERT_FALSE(BatchReader::IsEofBatch(read_batch)); @@ -2843,8 +3013,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its // location says that it is a format table, nor that everything below the location is data, so - // a path is not enough to reach it: the loaded table is handed to the context instead, which - // is what `FormatTable.newReadBuilder()` and `newBatchWriteBuilder()` do in Java. + // a path is not enough to reach it: operators must be created from the loaded table. std::unique_ptr schema_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(schema_dir); SchemaManager schema_manager(schema_dir->GetFileSystem(), schema_dir->Str()); @@ -2865,17 +3034,25 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // The same location by path alone is not a table at all: there is no schema under it to say // what kind of table it is. - ScanContextBuilder path_builder(data_dir->Str()); + TableLoadOptions path_builder_load_options{data_dir->Str()}; + std::map path_builder_options; + ScanContextBuilder path_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, path_builder.Finish()); - ASSERT_NOK(TableScan::Create(std::move(path_context))); + ASSERT_NOK(TableScan::Create(path_builder_load_options, std::move(path_context), + path_builder_options)); // Options given at the call still win over the ones the schema stored, so a file is rolled // per row rather than one file holding all three. - WriteContextBuilder write_builder(external); - write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); + const auto write_builder_table = external; + std::map write_builder_options; + WriteContextBuilder write_builder(""); + write_builder_options = {{Options::TARGET_FILE_ROW_NUM, "1"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_3, + write_builder_table->Copy(write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + configured_table_3->NewWrite(std::move(write_context))); for (int32_t i = 1; i <= 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {{"dt", "20240101"}})); @@ -2885,25 +3062,37 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - CommitContextBuilder commit_builder(external); + const auto commit_builder_table = external; + std::map commit_builder_options; + CommitContextBuilder commit_builder(""); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_4, + commit_builder_table->Copy(commit_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + configured_table_4->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - ScanContextBuilder scan_builder(external); + const auto scan_builder_table = external; + std::map scan_builder_options; + ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_5, + scan_builder_table->Copy(scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + configured_table_5->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions()); ASSERT_EQ(partitions, (PartitionList{{{"dt", "20240101"}}})); - ReadContextBuilder read_builder(external); + const auto read_builder_table = external; + std::map read_builder_options; + ReadContextBuilder read_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_6, + read_builder_table->Copy(read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + configured_table_6->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; while (true) { @@ -2919,67 +3108,9 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ASSERT_EQ(rows, 3); } -TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { - // The table already carries its schema and the file system it was loaded through, from a - // source the context cannot see behind, so a second answer is refused rather than dropped. - std::unique_ptr dir = UniqueTestDirectory::Create(); - ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - - ScanContextBuilder schema_builder(table); - schema_builder.SetTableSchema(schema_json); - ASSERT_NOK_WITH_MSG(schema_builder.Finish(), "carries its own schema"); - - ScanContextBuilder fs_builder(table); - fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(fs_builder.Finish(), "carries the file system"); - - ReadContextBuilder branch_builder(table); - branch_builder.WithBranch("other"); - ASSERT_NOK_WITH_MSG(branch_builder.Finish(), "has no branches"); - - WriteContextBuilder write_fs_builder(table); - write_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(write_fs_builder.Finish(), "carries the file system"); - - auto catalog = std::make_shared(); - WriteContextBuilder write_catalog_builder(table); - ASSERT_NOK_WITH_MSG( - write_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() requires a native table"); - ReadContextBuilder read_catalog_builder(table); - ASSERT_NOK_WITH_MSG(read_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - ScanContextBuilder scan_catalog_builder(table); - ASSERT_NOK_WITH_MSG(scan_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - catalog->SetTableSchema(latest.value()); - WriteContextBuilder path_catalog_builder(dir->Str(), "test-user"); - ASSERT_OK_AND_ASSIGN( - auto path_context, - path_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(path_context)), - "use WriteContextBuilder(FormatTable)"); - - CommitContextBuilder commit_fs_builder(table); - commit_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(commit_fs_builder.Finish(), "carries the file system"); - - // A null table is the caller's mistake, and says so rather than complaining about a path it - // was never given. - ScanContextBuilder null_builder{std::shared_ptr()}; - ASSERT_NOK_WITH_MSG(null_builder.Finish(), "null format table"); -} - TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) { - // `WithBranch()` is refused on a context built from a format table, and the `branch` option - // says the same thing by another name: a format table keeps no metadata to branch, and a read + // The `branch` option is refused when copying a format table: it keeps no metadata to branch, + // and a read // or a write would go to its one location whatever the option named. Refusing it is the only // answer that is not silently the wrong one - an overwrite would delete the main branch's // data while claiming to replace another branch's. @@ -2997,27 +3128,61 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); - ScanContextBuilder scan_builder(table); - scan_builder.SetOptions(branch); + const auto scan_builder_table = table; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = branch; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + scan_builder_table->Copy(scan_builder_options)); + return configured->NewScan(std::move(scan_context)); + })(), + "has no branches"); + + const auto read_builder_table = table; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = branch; - ReadContextBuilder read_builder(table); - read_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + read_builder_table->Copy(read_builder_options)); + return configured->NewRead(std::move(read_context)); + })(), + "has no branches"); + + const auto write_builder_table = table; + std::map write_builder_options; + WriteContextBuilder write_builder(""); + write_builder_options = branch; - WriteContextBuilder write_builder(table); - write_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + write_builder_table->Copy(write_builder_options)); + return configured->NewWrite(std::move(write_context)); + })(), + "has no branches"); // The commit is the one that matters most: an overwrite clears what it replaces before it // publishes anything, so the refusal has to come before the deleting does. - CommitContextBuilder commit_builder(table); - commit_builder.SetOptions(branch); + const auto commit_builder_table = table; + std::map commit_builder_options; + CommitContextBuilder commit_builder(""); + commit_builder_options = branch; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + commit_builder_table->Copy(commit_builder_options)); + return configured->NewCommit(std::move(commit_context)); + })(), + "has no branches"); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, @@ -3034,7 +3199,7 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) } TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { - // A context that names a path takes its branch from `WithBranch()`, and the schema of that + // A path load takes its branch from the dynamic `branch` option, and the schema of that // branch is what the entry point dispatches on. A format table's data is the files under the // table path whichever branch named it, so serving one from there under another branch's name // would answer a question about `dev` with the rows of `main`. @@ -3053,31 +3218,44 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr MakeSchema(), /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}})); - ReadContextBuilder read_builder(dir->Str()); - read_builder.WithBranch("dev"); + TableLoadOptions read_builder_load_options{dir->Str()}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options["branch"] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_builder_load_options, std::move(read_context), read_builder_options), + "has no branches"); // A scan takes its branch from the option rather than from a `WithBranch()` of its own, which // is the same branch by another name and gets the same answer. - ScanContextBuilder scan_builder(dir->Str()); - scan_builder.SetOptions({{Options::BRANCH, "dev"}}); + TableLoadOptions scan_builder_load_options{dir->Str()}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = {{Options::BRANCH, "dev"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG( + TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options), + "has no branches"); // The main branch is untouched by any of it. - ScanContextBuilder main_builder(dir->Str()); + TableLoadOptions main_builder_load_options{dir->Str()}; + std::map main_builder_options; + ScanContextBuilder main_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, - TableScan::Create(std::move(main_context))); + TableScan::Create(main_builder_load_options, std::move(main_context), + main_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, main_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector rows, ReadAll(table, plan->Splits())); ASSERT_EQ(rows, (std::vector{"1|alice|20240101"})); } TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { - // A context promises that options given at the call win over the ones the schema stored, and - // that a branch and a caller-held schema are honoured. Dispatching to a format table must not + // Static factories apply options over the stored schema and honor the requested branch and + // caller-supplied schema. Dispatching to a format table must not // quietly drop any of that. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -3086,11 +3264,15 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // `target-file-row-num` is stored nowhere in this table's schema, so a file would hold every // row. Given at the call it has to roll a file per row, as it does for a managed table. - WriteContextBuilder write_builder(dir->Str(), "test-user"); - write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); + TableLoadOptions write_builder_load_options{dir->Str()}; + std::map write_builder_options; + WriteContextBuilder write_builder("test-user"); + write_builder_options = {{Options::TARGET_FILE_ROW_NUM, "1"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); for (int32_t i = 0; i < 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {})); @@ -3102,20 +3284,28 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // Published through the generic commit, or nothing below would see the rows: until the commit // renames them the files sit in the hidden `_temporary` directory a scan skips. - CommitContextBuilder generic_commit_builder(dir->Str(), "test-user"); + TableLoadOptions generic_commit_builder_load_options{dir->Str()}; + std::map generic_commit_builder_options; + CommitContextBuilder generic_commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit_context, generic_commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit, - FileStoreCommit::Create(std::move(generic_commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr generic_commit, + FileStoreCommit::Create(generic_commit_builder_load_options, + std::move(generic_commit_context), generic_commit_builder_options)); ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); // A branch this table has no schema on is a managed table as far as dispatch is concerned, so // the format branch must not answer for it. - ReadContextBuilder branch_builder(dir->Str()); - branch_builder.WithBranch("nosuchbranch"); + TableLoadOptions branch_builder_load_options{dir->Str()}; + std::map branch_builder_options; + ReadContextBuilder branch_builder; + branch_builder_options["branch"] = "nosuchbranch"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); - ASSERT_NOK(TableRead::Create(std::move(branch_context))); + ASSERT_NOK(TableRead::Create(branch_builder_load_options, std::move(branch_context), + branch_builder_options)); // A caller-held schema is used instead of reading one from under the path. SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); @@ -3123,11 +3313,15 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { schema_manager.Latest()); ASSERT_TRUE(latest.has_value()); ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - ScanContextBuilder seeded_builder(dir->Str()); - seeded_builder.SetTableSchema(schema_json); + TableLoadOptions seeded_builder_load_options{dir->Str()}; + std::map seeded_builder_options; + ScanContextBuilder seeded_builder; + seeded_builder_load_options.table_schema_json = schema_json; + ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_context, seeded_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_scan, - TableScan::Create(std::move(seeded_context))); + TableScan::Create(seeded_builder_load_options, std::move(seeded_context), + seeded_builder_options)); // Planning, not just dispatch: handing the schema over must not turn the `schema` directory // the table keeps under its own path into data, which a plan would then read as a data file. ASSERT_OK_AND_ASSIGN(std::shared_ptr seeded_plan, seeded_scan->CreatePlan()); @@ -3138,11 +3332,15 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // `type` is structural, so an option given at the call must not decide what kind of table // this is. Checked by reading the rows back rather than by counting splits: three small files // are packed into one split, so a split count says nothing about how many there are. - ScanContextBuilder retyped_builder(dir->Str()); - retyped_builder.SetOptions({{Options::TYPE, "table"}}); + TableLoadOptions retyped_builder_load_options{dir->Str()}; + std::map retyped_builder_options; + ScanContextBuilder retyped_builder; + retyped_builder_options = {{Options::TYPE, "table"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_context, retyped_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_scan, - TableScan::Create(std::move(retyped_context))); + TableScan::Create(retyped_builder_load_options, std::move(retyped_context), + retyped_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr retyped_plan, retyped_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector retyped_rows, ReadAll(table, retyped_plan->Splits())); @@ -3189,15 +3387,17 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { MakeBatch({1, 2, 3}, {"alice", "bob", "carol"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(WriteAndCommit(table, std::move(batch))); - ScanContextBuilder scan_builder(dir->Str()); + TableLoadOptions scan_builder_load_options{dir->Str()}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - auto read_rows = [&plan](std::unique_ptr context) -> Result { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + auto read_rows = [&plan, &table](std::unique_ptr context) -> Result { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, table->NewRead(std::move(context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; @@ -3216,7 +3416,7 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { }; // Prefetch is honoured rather than refused, and reads back what was written. - ReadContextBuilder prefetch_builder(dir->Str()); + ReadContextBuilder prefetch_builder; prefetch_builder.EnablePrefetch(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr prefetch_context, prefetch_builder.Finish()); ASSERT_OK_AND_ASSIGN(int64_t prefetched_rows, read_rows(std::move(prefetch_context))); @@ -3224,7 +3424,7 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { // And the cache the context carries reaches the format reader. auto cache = std::make_shared(); - ReadContextBuilder cache_builder(dir->Str()); + ReadContextBuilder cache_builder; cache_builder.WithCache(cache); ASSERT_OK_AND_ASSIGN(std::unique_ptr cache_context, cache_builder.Finish()); ASSERT_OK_AND_ASSIGN(int64_t cached_rows, read_rows(std::move(cache_context))); @@ -3242,33 +3442,47 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { // A projected read schema can rename a column, prune a nested one and give it metadata of its // own; a format table's projection is a list of top-level names. - ReadContextBuilder read_schema_builder(dir->Str()); + TableLoadOptions read_schema_builder_load_options{dir->Str()}; + std::map read_schema_builder_options; + ReadContextBuilder read_schema_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> projected_schema, table->GetArrowSchema()); read_schema_builder.SetReadSchema(std::move(projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_schema_context, read_schema_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_schema_context)), - "does not take a projected read schema"); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_schema_builder_load_options, std::move(read_schema_context), + read_schema_builder_options), + "does not take a projected read schema"); - ScanContextBuilder streaming_builder(dir->Str()); + TableLoadOptions streaming_builder_load_options{dir->Str()}; + std::map streaming_builder_options; + ScanContextBuilder streaming_builder; streaming_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(streaming_builder_load_options, + std::move(streaming_context), streaming_builder_options), "nothing for a streaming scan to follow"); - ScanContextBuilder predicate_builder(dir->Str()); + TableLoadOptions predicate_builder_load_options{dir->Str()}; + std::map predicate_builder_options; + ScanContextBuilder predicate_builder; predicate_builder.SetPredicate(PredicateBuilder::GreaterThan( /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(1))); ASSERT_OK_AND_ASSIGN(std::unique_ptr predicate_context, predicate_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(predicate_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(predicate_builder_load_options, + std::move(predicate_context), predicate_builder_options), "does not take a predicate"); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); // A commit takes batch writes only. The rest of the snapshot half is // `TestTheGenericCommitRefusesEveryCallAboutSnapshots`. ASSERT_NOK_WITH_MSG(commit->Commit({}, /*commit_identifier=*/7), "commit identifier"); @@ -3291,10 +3505,14 @@ TEST(FormatTableTest, TestTheGenericCommitRefusesEveryCallAboutSnapshots) { ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress({}, BATCH_WRITE_COMMIT_IDENTIFIER, /*watermark=*/std::nullopt), @@ -3335,10 +3553,13 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { MakeBatch({9}, {"zoe"}, "20240102", {{"dt", "20240102"}})); ASSERT_OK(WriteAndCommit(table, std::move(other))); - WriteContextBuilder write_builder(dir->Str(), "test-user"); + TableLoadOptions write_builder_load_options{dir->Str()}; + std::map write_builder_options; + WriteContextBuilder write_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr replacement, MakeBatch({3}, {"carol"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(replacement))); @@ -3348,10 +3569,14 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 1u); - CommitContextBuilder commit_builder(dir->Str(), "test-user"); + TableLoadOptions commit_builder_load_options{dir->Str()}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); // Batch writes only, and refused before anything is cleared: an overwrite that failed on its // arguments must not have deleted the partition it named. ASSERT_NOK_WITH_MSG(commit->Overwrite({{"dt", "20240101"}}, messages, /*commit_identifier=*/7), @@ -5173,7 +5398,7 @@ TEST(FormatTableTest, TestWriteRejectsWrongPartitionSpec) { ASSERT_OK(write->Abort()); } -TEST(FormatTableTest, LoadedTableRejectsAnotherTablesWriteAndCommitContext) { +TEST(FormatTableTest, LoadedCommitOnlyChangesBoundTable) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -5181,74 +5406,31 @@ TEST(FormatTableTest, LoadedTableRejectsAnotherTablesWriteAndCommitContext) { CreateTable(fs, dir->Str() + "/a", {"dt"})); ASSERT_OK_AND_ASSIGN(std::shared_ptr second, CreateTable(fs, dir->Str() + "/b", {"dt"})); - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, - MakeBatch({1}, {"alice"}, "p", {{"dt", "p"}})); - ASSERT_OK(WriteAndCommit(first, std::move(batch))); + for (const auto& table : {first, second}) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch({1}, {"alice"}, "p", {{"dt", "p"}})); + ASSERT_OK(WriteAndCommit(table, std::move(batch))); + } std::vector before; - ASSERT_OK(fs->ListFileStatus(first->Location() + "/dt=p", &before)); + ASSERT_OK(fs->ListFileStatus(second->Location() + "/dt=p", &before)); before.erase(std::remove_if(before.begin(), before.end(), [](const FileStatus& file) { return file.IsDir(); }), before.end()); ASSERT_FALSE(before.empty()); - for (bool typed : {false, true}) { - std::unique_ptr commit_context; - std::unique_ptr write_context; - if (typed) { - ASSERT_OK_AND_ASSIGN(commit_context, CommitContextBuilder(second).Finish()); - ASSERT_OK_AND_ASSIGN(write_context, WriteContextBuilder(second).Finish()); - } else { - ASSERT_OK_AND_ASSIGN(commit_context, - CommitContextBuilder(second->Location(), "user").Finish()); - ASSERT_OK_AND_ASSIGN(write_context, - WriteContextBuilder(second->Location(), "user").Finish()); - } - ASSERT_NOK_WITH_MSG(first->NewCommit(std::move(commit_context)), - "context path contradicts"); - ASSERT_NOK_WITH_MSG(first->NewWrite(std::move(write_context)), "context path contradicts"); - for (const auto& file : before) { - ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists(file.GetPath())); - ASSERT_TRUE(exists); - } - } - ASSERT_OK_AND_ASSIGN( - std::shared_ptr other_identity, - FormatTable::Create(fs, first->Location(), Identifier("other_db", "tbl"), {})); - ASSERT_OK_AND_ASSIGN(std::unique_ptr wrong_identity, - CommitContextBuilder(other_identity).Finish()); - ASSERT_NOK_WITH_MSG(first->NewCommit(std::move(wrong_identity)), - "context identity contradicts"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr valid, - CommitContextBuilder(first).Finish()); + // An execution context has no second table identity to redirect the commit. + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + CommitContextBuilder("user").Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - first->NewCommit(std::move(valid))); + first->NewCommit(std::move(context))); ASSERT_OK(commit->Overwrite({{"dt", "p"}}, {}, BATCH_WRITE_COMMIT_IDENTIFIER)); for (const auto& file : before) { ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists(file.GetPath())); - ASSERT_FALSE(exists); + ASSERT_TRUE(exists); } -} - -TEST(FormatTableTest, LoadedWriteContextsRequireSameStorageBindingAndAcceptMainAlias) { - auto dir = UniqueTestDirectory::Create(); - ASSERT_TRUE(dir); - auto fs = dir->GetFileSystem(); - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(fs, dir->Str(), {"dt"})); - auto other_fs = std::make_shared(fs); - ASSERT_OK_AND_ASSIGN(std::shared_ptr other, - FormatTable::Create(other_fs, dir->Str(), Identifier("db", "tbl"), {})); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write, WriteContextBuilder(other).Finish()); - ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write)), "context filesystem contradicts"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - CommitContextBuilder(other).Finish()); - ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit)), "context filesystem contradicts"); - ASSERT_TRUE(other_fs->Calls().empty()); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr alias, - FormatTable::Create(fs, dir->Str(), Identifier("db", "tbl$branch_MAIN"), {})); - ASSERT_OK_AND_ASSIGN(write, WriteContextBuilder(alias).Finish()); - ASSERT_OK(table->NewWrite(std::move(write))); - ASSERT_OK_AND_ASSIGN(commit, CommitContextBuilder(alias).Finish()); - ASSERT_OK(table->NewCommit(std::move(commit))); + std::vector remaining; + ASSERT_OK(fs->ListFileStatus(first->Location() + "/dt=p", &remaining)); + ASSERT_TRUE(std::all_of(remaining.begin(), remaining.end(), + [](const FileStatus& file) { return file.IsDir(); })); } } // namespace paimon::test diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp index 8fd31c4c8..7ce13b431 100644 --- a/src/paimon/core/table/sink/commit_message_test.cpp +++ b/src/paimon/core/table/sink/commit_message_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/linked_hash_map.h" @@ -1230,16 +1231,17 @@ TEST(CommitMessageTest, TestSerialize) { /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), - "commit_user1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.AddOption(Options::FILE_FORMAT, "mock_format") - .AddOption(Options::TARGET_FILE_SIZE, "1024") - .Finish()); + TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user1"); + context_builder_options[Options::FILE_FORMAT] = "mock_format"; + context_builder_options[Options::TARGET_FILE_SIZE] = "1024"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - std::string root_path = write_context->GetRootPath(); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); for (size_t i = 0; i < 10240; i++) { auto f0_array = std::make_shared(); diff --git a/src/paimon/core/table/source/append_count_reader_test.cpp b/src/paimon/core/table/source/append_count_reader_test.cpp index 3b191a3d0..1814f2b39 100644 --- a/src/paimon/core/table/source/append_count_reader_test.cpp +++ b/src/paimon/core/table/source/append_count_reader_test.cpp @@ -25,6 +25,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" @@ -51,14 +52,18 @@ class AppendCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; if (snapshot_id.has_value()) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); if (snapshot_id.has_value() && (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id)) { @@ -133,10 +138,14 @@ TEST_F(AppendCountReaderTest, TestAppendOnlyTableReadCreateCountReaderPredicateN auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(splits), diff --git a/src/paimon/core/table/source/filtered_row_id_reader.cpp b/src/paimon/core/table/source/filtered_row_id_reader.cpp index 6c7d54f1a..5742cc4d0 100644 --- a/src/paimon/core/table/source/filtered_row_id_reader.cpp +++ b/src/paimon/core/table/source/filtered_row_id_reader.cpp @@ -30,6 +30,7 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "fmt/format.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -100,16 +101,17 @@ Result> PinToSnapshot(const std::string& root /// `FilteredRowIdReader` reads with the table that the search loaded, so that a column renamed or /// replaced since then cannot point the filter at another field. Result> CreateReadWithSchema( - std::unique_ptr read_context, const std::shared_ptr& table_schema, + const TableLoadOptions& load, std::unique_ptr read_context, + const std::shared_ptr& table_schema, const std::map& options, const std::shared_ptr& pool, const std::shared_ptr& executor) { // Row ids come from row tracking, which a primary-key table does not support. if (!table_schema->PrimaryKeys().empty()) { return Status::NotImplemented("cannot evaluate a row filter on a primary-key table"); } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(std::move(read_context), table_schema, + options, load.file_system, load.path)); const CoreOptions& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, @@ -167,16 +169,19 @@ Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap6 // Unlike the read, the scan may load the latest schema, as it selects files by partition and // row id alone. - ScanContextBuilder scan_builder(root_path_); - scan_builder.SetOptions(pinned_options) - .SetPredicate(partition_filter_) + TableLoadOptions scan_builder_load{root_path_}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = pinned_options; + scan_builder_load.file_system = options_.GetFileSystem(); + scan_builder.SetPredicate(partition_filter_) .SetGlobalIndexResult(std::make_shared(rows)) - .WithFileSystem(options_.GetFileSystem()) .WithMemoryPool(pool_) .WithExecutor(executor_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(scan_builder_load, std::move(scan_context), scan_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); // An empty plan of another snapshot would drop every candidate, so it is checked too. if (plan->SnapshotId() != snapshot_id_) { @@ -202,19 +207,21 @@ Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap6 PredicateUtils::CreatePickedFieldFilter( filter_, ObjectUtils::CreateIdentifierToIndexMap(read_fields))); - ReadContextBuilder read_builder(root_path_); - read_builder.SetOptions(pinned_options) - .SetReadFieldNames(read_fields) + TableLoadOptions read_builder_load{root_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = pinned_options; + read_builder_load.file_system = options_.GetFileSystem(); + read_builder.SetReadFieldNames(read_fields) .SetPredicate(read_filter) .EnablePredicateFilter(true) .EnablePrefetch(true) - .WithFileSystem(options_.GetFileSystem()) .WithMemoryPool(pool_) .WithExecutor(executor_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - CreateReadWithSchema(std::move(read_context), table_schema_, - pinned_options, pool_, executor_)); + CreateReadWithSchema(read_builder_load, std::move(read_context), + table_schema_, pinned_options, pool_, executor_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ScopeGuard guard([&reader]() { reader->Close(); }); diff --git a/src/paimon/core/table/source/pk_count_reader_test.cpp b/src/paimon/core/table/source/pk_count_reader_test.cpp index f9ce137ef..832e7125f 100644 --- a/src/paimon/core/table/source/pk_count_reader_test.cpp +++ b/src/paimon/core/table/source/pk_count_reader_test.cpp @@ -25,6 +25,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/types/data_field.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/core/schema/schema_manager.h" @@ -51,14 +52,18 @@ class PKCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; if (snapshot_id.has_value()) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); if (snapshot_id.has_value()) { if (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id) { @@ -70,7 +75,9 @@ class PKCountReaderTest : public testing::Test { Result> CreateInternalContext( const std::string& table_path) { - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, read_context_builder.Finish()); @@ -78,7 +85,13 @@ class PKCountReaderTest : public testing::Test { PAIMON_ASSIGN_OR_RAISE(auto table_schema, schema_manager.ReadSchema(0)); PAIMON_ASSIGN_OR_RAISE( auto internal_context, - InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + InternalReadContext::Create( + read_context, table_schema, table_schema->Options(), + read_context_builder_load_options.file_system, + read_context_builder_load_options.path, + read_context_builder_load_options.fs_scheme_to_identifier_map, + read_context_builder_load_options.catalog, + read_context_builder_load_options.identifier, read_context_builder_options)); return std::shared_ptr(std::move(internal_context)); } diff --git a/src/paimon/core/table/source/query_auth_table_read.cpp b/src/paimon/core/table/source/query_auth_table_read.cpp index b004ddaab..98ae708b1 100644 --- a/src/paimon/core/table/source/query_auth_table_read.cpp +++ b/src/paimon/core/table/source/query_auth_table_read.cpp @@ -232,6 +232,7 @@ Result QueryAuthBatchReader::NextBatch() { Result> QueryAuthTableRead::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, + const std::string& table_path, const std::string& branch, const std::shared_ptr& arrow_pool, InnerReadFactory create_inner) { if (context->HasVectorOrFullTextSearch() || context->EnablePreciseFileIndexEvaluate()) { return Status::NotImplemented( @@ -278,8 +279,9 @@ Result> QueryAuthTableRead::Create( name)); } } - std::unique_ptr read(new QueryAuthTableRead( - context, table_schema, arrow_pool, std::move(create_inner), std::move(output_fields))); + std::unique_ptr read( + new QueryAuthTableRead(context, table_schema, table_path, branch, arrow_pool, + std::move(create_inner), std::move(output_fields))); // Built now, so a read the table cannot serve fails here rather than at the first split. PAIMON_RETURN_NOT_OK(read->GetOrCreateInnerRead({}).status()); return std::unique_ptr(std::move(read)); @@ -287,10 +289,13 @@ Result> QueryAuthTableRead::Create( QueryAuthTableRead::QueryAuthTableRead(const std::shared_ptr& context, const std::shared_ptr& table_schema, + const std::string& table_path, const std::string& branch, const std::shared_ptr& arrow_pool, InnerReadFactory create_inner, std::optional> output_fields) : context_(context), + table_path_(table_path), + branch_(branch), table_schema_(table_schema), arrow_pool_(arrow_pool), create_inner_(std::move(create_inner)), @@ -337,17 +342,15 @@ Result QueryAuthTableRead::GetOrCreateInnerRead( } // Without the predicate: file statistics and indexes hold raw values. auto inner_context = std::make_unique( - context_->GetPath(), context_->GetBranch(), read_field_names, read_field_ids, + read_field_names, read_field_ids, /*predicate=*/nullptr, /*vector_search=*/nullptr, /*full_text_search=*/nullptr, /*enable_precise_file_index_evaluate=*/false, /*enable_predicate_filter=*/false, context_->EnablePrefetch(), context_->EnableLateMaterializing(), context_->GetPrefetchBatchCount(), context_->GetPrefetchMaxParallelNum(), context_->EnableMultiThreadRowToBatch(), context_->GetRowToBatchThreadNumber(), - context_->GetSpecificTableSchema(), context_->GetMemoryPool(), context_->GetExecutor(), - context_->GetSpecificFileSystem(), context_->GetFileSystemSchemeToIdentifierMap(), - /*realtime_context=*/nullptr, context_->GetOptions(), context_->ReadAheadCacheEnabled(), - context_->GetCacheConfig(), context_->GetCache(), context_->GetFormatTable(), - context_->GetWarmupLevel(), context_->GetCatalog(), context_->GetIdentifier()); + context_->GetMemoryPool(), context_->GetExecutor(), + /*realtime_context=*/nullptr, context_->ReadAheadCacheEnabled(), context_->GetCacheConfig(), + context_->GetCache(), context_->GetWarmupLevel()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, create_inner_(std::move(inner_context))); TableRead* raw_read = read.get(); @@ -370,13 +373,13 @@ Status QueryAuthTableRead::ValidateSplit(const QueryAuthSplit& split, "authorized for. Plan the scan again."); } PAIMON_ASSIGN_OR_RAISE(bool same_location, - CatalogUtils::SameTableLocation(scope.table_path, context_->GetPath())); - const std::string read_branch = BranchManager::NormalizeBranch(context_->GetBranch()); + CatalogUtils::SameTableLocation(scope.table_path, table_path_)); + const std::string read_branch = BranchManager::NormalizeBranch(branch_); if (!same_location || scope.branch != read_branch) { return Status::Invalid(fmt::format( "The split was authorized for table {} (branch {}), but the read targets {} (branch " "{}). Plan the scan for the table and branch being read.", - scope.table_path, scope.branch, context_->GetPath(), read_branch)); + scope.table_path, scope.branch, table_path_, read_branch)); } const TableQueryAuthResult& rules = *split.GetAuthResult(); const std::vector& fields = table_schema_->Fields(); @@ -425,9 +428,8 @@ Result> QueryAuthTableRead::CreateReader( } Status valid = ValidateSplit(*auth_split, filter_fields); if (!valid.ok()) { - return valid.WithMessage(fmt::format("Query auth read of {} (schema {}): {}", - context_->GetPath(), table_schema_->Id(), - valid.message())); + return valid.WithMessage(fmt::format("Query auth read of {} (schema {}): {}", table_path_, + table_schema_->Id(), valid.message())); } if (rules.RejectsAllRows()) { diff --git a/src/paimon/core/table/source/query_auth_table_read.h b/src/paimon/core/table/source/query_auth_table_read.h index 47c396285..7796d6994 100644 --- a/src/paimon/core/table/source/query_auth_table_read.h +++ b/src/paimon/core/table/source/query_auth_table_read.h @@ -101,8 +101,9 @@ class QueryAuthTableRead : public TableRead { /// @param create_inner Builds the read the rules are applied around. static Result> Create( const std::shared_ptr& context, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_pool, InnerReadFactory create_inner); + const std::shared_ptr& table_schema, const std::string& table_path, + const std::string& branch, const std::shared_ptr& arrow_pool, + InnerReadFactory create_inner); Result> CreateReader(const std::shared_ptr& split) override; Result> CreateReader( @@ -111,6 +112,7 @@ class QueryAuthTableRead : public TableRead { private: QueryAuthTableRead(const std::shared_ptr& context, const std::shared_ptr& table_schema, + const std::string& table_path, const std::string& branch, const std::shared_ptr& arrow_pool, InnerReadFactory create_inner, std::optional> output_fields); @@ -129,6 +131,8 @@ class QueryAuthTableRead : public TableRead { Result GetOrCreateInnerRead(const std::vector& extra_fields); std::shared_ptr context_; + std::string table_path_; + std::string branch_; std::shared_ptr table_schema_; std::shared_ptr arrow_pool_; InnerReadFactory create_inner_; diff --git a/src/paimon/core/table/source/query_auth_table_read_test.cpp b/src/paimon/core/table/source/query_auth_table_read_test.cpp index 32e39d7f4..afa9c4811 100644 --- a/src/paimon/core/table/source/query_auth_table_read_test.cpp +++ b/src/paimon/core/table/source/query_auth_table_read_test.cpp @@ -253,15 +253,14 @@ TEST(QueryAuthTableReadTest, TestClosesTheReadersOfSplitsBeforeOneThatFails) { /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::QUERY_AUTH_ENABLED, "true"}})); std::shared_ptr table_schema = std::move(created); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder("table_root_path").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); int32_t closes = 0; auto failing_split = std::make_shared(); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, QueryAuthTableRead::Create( - std::shared_ptr(std::move(context)), table_schema, - GetArrowPool(GetDefaultPool()), + std::shared_ptr(std::move(context)), table_schema, "table_root_path", + "main", GetArrowPool(GetDefaultPool()), CreateSplitTableRead([&closes, failing_split](const std::shared_ptr& split) -> Result> { if (split == failing_split) { @@ -351,13 +350,12 @@ TEST(QueryAuthTableReadTest, TestRowFilterRemovesRowsFromTheBatchWithBitmap) { scope.authorized_field_ids.emplace(field.Name(), field.Id()); } scope.rule_field_ids = rules->BindFieldIds(table_schema->Fields()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder("table_root_path").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, QueryAuthTableRead::Create( - std::shared_ptr(std::move(context)), table_schema, - GetArrowPool(GetDefaultPool()), + std::shared_ptr(std::move(context)), table_schema, "table_root_path", + "main", GetArrowPool(GetDefaultPool()), CreateSplitTableRead( [batch](const std::shared_ptr&) -> Result> { return std::unique_ptr(std::make_unique(batch)); diff --git a/src/paimon/core/table/source/query_auth_table_scan.cpp b/src/paimon/core/table/source/query_auth_table_scan.cpp index e1c72b29d..f6d9f08d9 100644 --- a/src/paimon/core/table/source/query_auth_table_scan.cpp +++ b/src/paimon/core/table/source/query_auth_table_scan.cpp @@ -84,14 +84,15 @@ std::string FieldIdOrAbsent(const DataField* field) { Result> QueryAuthTableScan::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::shared_ptr& file_system, const std::string& branch, - InnerScanFactory create_inner) { - if (context->GetCatalog() == nullptr || !context->GetIdentifier()) { + const std::shared_ptr& catalog, const Identifier& identifier, + const std::string& table_path, InnerScanFactory create_inner) { + if (catalog == nullptr) { return Status::NotImplemented(fmt::format( "Cannot read a table with '{}=true' without the catalog that serves its row filters " - "and column masks: build the scan with ScanContextBuilder::WithCatalog().", + "and column masks: load the table through its catalog.", Options::QUERY_AUTH_ENABLED)); } - const QueryAuthCatalog* auth_catalog = AsQueryAuthCatalog(context->GetCatalog()); + const QueryAuthCatalog* auth_catalog = AsQueryAuthCatalog(catalog); if (auth_catalog == nullptr) { return Status::NotImplemented(fmt::format( "Cannot read a table with '{}=true': its catalog does not support the query auth " @@ -108,14 +109,11 @@ Result> QueryAuthTableScan::Create( } // The rules are asked for the branch planned, even when only the options name it. PAIMON_ASSIGN_OR_RAISE(std::string resolved_branch, - BranchManager::ResolveBranch(context->GetIdentifier(), - context->GetOptions(), branch, "scan")); - PAIMON_ASSIGN_OR_RAISE( - Identifier auth_identifier, - CatalogUtils::ResolvedBranchIdentifier(context->GetIdentifier().value(), resolved_branch)); + BranchManager::ResolveBranch(identifier, {}, branch, "scan")); + PAIMON_ASSIGN_OR_RAISE(Identifier auth_identifier, + CatalogUtils::ResolvedBranchIdentifier(identifier, resolved_branch)); PAIMON_RETURN_NOT_OK(CatalogUtils::CheckPathIsTableLocation( - auth_identifier, context->GetCatalog()->GetTableLocation(auth_identifier), - context->GetPath(), "scan")); + auth_identifier, catalog->GetTableLocation(auth_identifier), table_path, "scan")); std::set user_filter_fields; std::set partition_filter_fields; const std::shared_ptr& scan_filter = context->GetScanFilters(); @@ -153,19 +151,23 @@ Result> QueryAuthTableScan::Create( if (read_field_names) { authorization_columns = std::move(columns); } - return std::unique_ptr(new QueryAuthTableScan( - context, auth_catalog, std::move(auth_identifier), table_schema, file_system, - resolved_branch, std::move(create_inner), std::move(authorization_columns), - std::move(user_filter_fields), std::move(partition_filter_fields))); + return std::unique_ptr( + new QueryAuthTableScan(context, auth_catalog, std::move(auth_identifier), table_schema, + file_system, resolved_branch, catalog, table_path, + std::move(create_inner), std::move(authorization_columns), + std::move(user_filter_fields), std::move(partition_filter_fields))); } QueryAuthTableScan::QueryAuthTableScan( const std::shared_ptr& context, const QueryAuthCatalog* auth_catalog, Identifier auth_identifier, const std::shared_ptr& table_schema, const std::shared_ptr& file_system, const std::string& branch, + const std::shared_ptr& catalog, const std::string& table_path, InnerScanFactory create_inner, std::optional> authorization_columns, std::set user_filter_fields, std::set partition_filter_fields) : context_(context), + catalog_(catalog), + table_path_(table_path), auth_catalog_(auth_catalog), auth_identifier_(std::move(auth_identifier)), table_schema_(table_schema), @@ -187,7 +189,7 @@ Status QueryAuthTableScan::AddErrorContext(const Status& status) const { } Status QueryAuthTableScan::ValidateAgainstCurrentSchemas(const TableQueryAuthResult& rules) const { - SchemaManager schema_manager(file_system_, context_->GetPath(), branch_); + SchemaManager schema_manager(file_system_, table_path_, branch_); PAIMON_ASSIGN_OR_RAISE(std::optional> latest, schema_manager.Latest()); const std::shared_ptr& latest_schema = latest ? latest.value() : table_schema_; @@ -232,8 +234,8 @@ Status QueryAuthTableScan::ValidateAgainstCurrentSchemas(const TableQueryAuthRes PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog_schema, auth_catalog_->LoadTableSchemaAndLocation(auth_identifier_, &location)); // The identifier may have been bound to another table since the scan was created. - PAIMON_RETURN_NOT_OK(CatalogUtils::CheckPathIsTableLocation(auth_identifier_, location, - context_->GetPath(), "scan")); + PAIMON_RETURN_NOT_OK( + CatalogUtils::CheckPathIsTableLocation(auth_identifier_, location, table_path_, "scan")); auto catalog_data_schema = std::dynamic_pointer_cast(catalog_schema); if (catalog_data_schema == nullptr) { return Status::Invalid("Query auth requires a data table schema from the catalog."); @@ -318,13 +320,10 @@ Result> QueryAuthTableScan::CreateInnerScan( // As in Java, a row filter on data columns turns the limit off too: it is pushed down as a // predicate, which the inner scan does not push a limit past. auto inner_context = std::make_shared( - context_->GetPath(), context_->IsStreamingMode(), + context_->IsStreamingMode(), masked_conjuncts_excluded ? std::optional() : context_->GetLimit(), pushed_filter, /*global_index_result=*/nullptr, /*realtime_context=*/nullptr, context_->GetMemoryPool(), - context_->GetExecutor(), context_->GetSpecificFileSystem(), - context_->GetSpecificTableSchema(), context_->GetOptions(), context_->GetCache(), - context_->GetFormatTable(), context_->GetCatalog(), context_->GetIdentifier(), - context_->GetReadFieldNames()); + context_->GetExecutor(), context_->GetCache(), context_->GetReadFieldNames()); return create_inner_(inner_context); } @@ -362,7 +361,7 @@ Result> QueryAuthTableScan::CreatePlan() { // Bound to the schema planned, so the read can tell a column since renamed or replaced, and // one added since, from the columns the rules were authorized for. QueryAuthScope scope; - scope.table_path = context_->GetPath(); + scope.table_path = table_path_; scope.branch = branch_; for (const DataField& field : table_schema_->Fields()) { if (!authorization_columns_ || diff --git a/src/paimon/core/table/source/query_auth_table_scan.h b/src/paimon/core/table/source/query_auth_table_scan.h index 0f7d9a193..55c1d9faa 100644 --- a/src/paimon/core/table/source/query_auth_table_scan.h +++ b/src/paimon/core/table/source/query_auth_table_scan.h @@ -49,12 +49,14 @@ class QueryAuthTableScan : public TableScan { using InnerScanFactory = std::function>(const std::shared_ptr&)>; - /// @param context The scan as the caller asked for it; it must name the catalog and the table. + /// @param context The execution settings for the scan. + /// @param catalog The catalog retained by the loaded table. static Result> Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::shared_ptr& file_system, const std::string& branch, - InnerScanFactory create_inner); + const std::shared_ptr& catalog, const Identifier& identifier, + const std::string& table_path, InnerScanFactory create_inner); Result> CreatePlan() override; @@ -65,6 +67,7 @@ class QueryAuthTableScan : public TableScan { const QueryAuthCatalog* auth_catalog, Identifier auth_identifier, const std::shared_ptr& table_schema, const std::shared_ptr& file_system, const std::string& branch, + const std::shared_ptr& catalog, const std::string& table_path, InnerScanFactory create_inner, std::optional> authorization_columns, std::set user_filter_fields, @@ -90,8 +93,10 @@ class QueryAuthTableScan : public TableScan { /// Fails when `rules` would need pruning the inner scan already pushed down to be undone. Status ValidatePushdownCompatibility(const TableQueryAuthResult& rules) const; - /// Holds the catalog `auth_catalog_` points to. std::shared_ptr context_; + /// Holds the catalog `auth_catalog_` points to. + std::shared_ptr catalog_; + std::string table_path_; const QueryAuthCatalog* auth_catalog_; /// The table the rules are asked for, naming the branch planned. Identifier auth_identifier_; diff --git a/src/paimon/core/table/source/scan_exception_safety_test.cpp b/src/paimon/core/table/source/scan_exception_safety_test.cpp index 246ab10f7..c453a6933 100644 --- a/src/paimon/core/table/source/scan_exception_safety_test.cpp +++ b/src/paimon/core/table/source/scan_exception_safety_test.cpp @@ -30,6 +30,7 @@ #include "arrow/memory_pool.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -117,18 +118,21 @@ Result> OrdinaryScan( std::ifstream schema_file(input.path + "/schema/schema-0"); std::string schema{std::istreambuf_iterator(schema_file), std::istreambuf_iterator()}; - ScanContextBuilder builder(input.path); - builder.SetTableSchema(schema).WithMemoryPool(pool).WithExecutor(executor).AddOption( - "scan.snapshot-id", input.snapshot); + TableLoadOptions builder_load_options{input.path}; + std::map builder_options; + ScanContextBuilder builder; + builder_load_options.table_schema_json = schema; + builder_options["scan.snapshot-id"] = input.snapshot; + builder.WithMemoryPool(pool).WithExecutor(executor); if (input.cache) { - builder.WithCache(input.cache) - .AddOption("scan.manifest-entry-cache.max-snapshots", "8") - .AddOption("scan.manifest-entry.lazy-decode.enabled", "true"); + builder_options["scan.manifest-entry-cache.max-snapshots"] = "8"; + builder_options["scan.manifest-entry.lazy-decode.enabled"] = "true"; + builder.WithCache(input.cache); } if (input.bucket) builder.SetBucketFilter(*input.bucket); - if (input.file_system) builder.WithFileSystem(input.file_system); + builder_load_options.file_system = input.file_system; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return TableScan::Create(std::move(context)); + return TableScan::Create(builder_load_options, std::move(context), builder_options); } class RejectingPlanningExecutor final : public Executor { diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3433cd670..e344a4a46 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -32,6 +32,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/catalog/catalog_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/core/core_options.h" @@ -65,71 +66,92 @@ TEST(TableReadTest, TestReadWithInvalidContext) { std::string path = paimon::test::GetDataDir() + "/orc/append_10.db/append_10/"; { // read with non-exist field - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "non-exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "Get field non-exist failed: not exist in table schema"); } { // field type and literal type mismatch auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(int64_t{15})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(std::move(read_context)), + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "field f3 has field type BIGINT in literal, mismatch field type DOUBLE in predicate"); } { // field type in predicate mismatch schema auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BIGINT, Literal(int64_t{15})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "schema type double mismatches predicate field type BIGINT"); } { // field idx in predicate mismatch in schema auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f3", FieldType::DOUBLE, Literal(15.0)); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(std::move(read_context)), + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "field f3 has field idx 0 in input schema, mismatch field idx 2 in predicate"); } { // literal cannot be null auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(FieldType::DOUBLE)); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "literal cannot be null in predicate, field name f3"); } { // schema with duplicate field f3 - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f1", "f3"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "validate schema failed: read schema has duplicate field f3"); } } TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"field_no_exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options), "Get field field_no_exist failed: not exist in table schema"); } @@ -147,20 +169,45 @@ TEST(TableReadTest, TestFullTextMatchReadEntryAndScoreValidation) { FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), "", "main")); for (bool table_entry : {false, true}) { - ReadContextBuilder builder(table_path); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0", "f1"}) .SetPredicate(predicate); + auto context_load_options = builder_load_options; + auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK(table_entry ? table->NewRead(std::move(context)) - : TableRead::Create(std::move(context))); + ASSERT_OK(table_entry ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, table->Copy(context_options)); + return configured->NewRead(std::move(context)); + })() + : TableRead::Create(context_load_options, std::move(context), + context_options)); builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}) .SetPredicate(predicate); + context_load_options.path = builder_load_options.path; + context_load_options.file_system = builder_load_options.file_system; + context_load_options.fs_scheme_to_identifier_map = + builder_load_options.fs_scheme_to_identifier_map; + context_load_options.table_schema_json = builder_load_options.table_schema_json; + context_load_options.catalog = builder_load_options.catalog; + context_load_options.table_id = builder_load_options.table_id; + context_load_options.identifier.reset(); + if (builder_load_options.identifier) { + context_load_options.identifier.emplace(*builder_load_options.identifier); + } + context_options = builder_options; ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - ASSERT_NOK_WITH_MSG(table_entry ? table->NewRead(std::move(context)) - : TableRead::Create(std::move(context)), + ASSERT_NOK_WITH_MSG(table_entry ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(context_options)); + return configured->NewRead(std::move(context)); + })() + : TableRead::Create(context_load_options, + std::move(context), context_options), "Score-producing FullTextMatch cannot appear under OR"); } } @@ -177,23 +224,33 @@ TEST(TableReadTest, TestPerFieldScoreReadEntry) { FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), "", "main")); for (bool table_entry : {false, true}) { - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(search).SetReadFieldNames( {"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK(table_entry ? table->NewRead(std::move(context)) - : TableRead::Create(std::move(context))); + ASSERT_OK(table_entry ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, table->Copy(builder_options)); + return configured->NewRead(std::move(context)); + })() + : TableRead::Create(builder_load_options, std::move(context), + builder_options)); } } TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) { const std::string path = GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; - ScanContextBuilder scan_builder(path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); + TableLoadOptions scan_builder_load_options{path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); std::shared_ptr selected_split; for (const auto& split : plan->Splits()) { @@ -222,13 +279,17 @@ TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) auto scalar = PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0)); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({text, scalar})); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options[Options::READ_BATCH_SIZE] = "1"; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"_f0_INDEX_SCORE", "f3", "f0"}) - .SetPredicate(predicate) - .AddOption(Options::READ_BATCH_SIZE, "1"); + .SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(selected_split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -247,12 +308,18 @@ TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) TEST(TableReadTest, TestCreateKeyValueTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); auto key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read); } @@ -262,12 +329,16 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; auto search = std::make_shared("f0", R"({"match":{"query":"Alice"}})", /*limit=*/1); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0"}) .SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "CreateCountReader does not support precise file-index evaluation"); } @@ -277,12 +348,16 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { auto search = std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0"}) .SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "CreateCountReader does not support precise file-index evaluation"); } @@ -290,10 +365,13 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { TEST(TableReadTest, TestPreciseEvaluationRejectsSystemAndFormatTables) { std::string path = GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder system_builder(path + "$snapshots"); + TableLoadOptions system_builder_load_options{path + "$snapshots"}; + std::map system_builder_options; + ReadContextBuilder system_builder; system_builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto system_context, system_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(system_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(system_builder_load_options, std::move(system_context), + system_builder_options), "Precise file-index evaluation is not supported for system tables"); auto directory = UniqueTestDirectory::Create(); @@ -306,11 +384,14 @@ TEST(TableReadTest, TestPreciseEvaluationRejectsSystemAndFormatTables) { FormatTable::Create(directory->GetFileSystem(), directory->Str(), Identifier("db", "table"), {})); for (bool loaded_table : {false, true}) { - ReadContextBuilder builder = - loaded_table ? ReadContextBuilder(format_table) : ReadContextBuilder(directory->Str()); + ReadContextBuilder builder; + TableLoadOptions builder_load_options{directory->Str()}; + std::map builder_options; builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), + ASSERT_NOK_WITH_MSG(loaded_table + ? format_table->NewRead(std::move(context)) + : TableRead::Create(builder_load_options, std::move(context)), "Precise file-index evaluation is not supported for format tables"); } } @@ -321,13 +402,16 @@ TEST(TableReadTest, TestPreciseScalarEvaluationRejectsUnsupportedTableModes) { GetDataDir() + (primary_key ? "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality" : "/orc/append_09.db/append_09"); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f3"}).SetPredicate( PredicateBuilder::GreaterThan(0, "f3", FieldType::DOUBLE, Literal(0.0))); if (primary_key) { - builder.AddOption(Options::DELETION_VECTORS_ENABLED, "false"); + builder_options[Options::DELETION_VECTORS_ENABLED] = "false"; + } else { - builder.AddOption(Options::DATA_EVOLUTION_ENABLED, "true"); + builder_options[Options::DATA_EVOLUTION_ENABLED] = "true"; } ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); const std::string expected_error = @@ -335,7 +419,9 @@ TEST(TableReadTest, TestPreciseScalarEvaluationRejectsUnsupportedTableModes) { "deletion vectors" : "Precise file-index evaluation does not support data-evolution or " "real-time reads"; - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), expected_error); + ASSERT_NOK_WITH_MSG( + TableRead::Create(builder_load_options, std::move(context), builder_options), + expected_error); } } @@ -345,11 +431,15 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { auto search = std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f0"}).SetVectorSearch( search); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + TableRead::Create(context_builder_load_options, std::move(context), + context_builder_options)); BinaryRow partition = BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get()); DataSplitImpl::Builder builder(partition, /*bucket=*/0, /*bucket_path=*/"", /*data_files=*/{}); @@ -369,26 +459,40 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { TEST(TableReadTest, TestCreateAppendOnlyTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); } TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); - context_builder.AddOption("bucket", "10"); - context_builder.AddOption("bucket-key", "f3"); + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["orc.read.enable-lazy-decoding"] = "true"; + + context_builder_options["bucket"] = "10"; + + context_builder_options["bucket-key"] = "f3"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); auto* typed_split_read = @@ -413,15 +517,19 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { ASSERT_OK_AND_ASSIGN(std::string schema_json, table_schema->ToJsonString()); for (bool explicit_header : {false, true}) { - ReadContextBuilder builder("warehouse/db.db/table"); - builder.SetTableSchema(schema_json); + TableLoadOptions builder_load_options{"warehouse/db.db/table"}; + std::map builder_options; + ReadContextBuilder builder; + builder_load_options.table_schema_json = schema_json; + if (explicit_header) { - builder.AddOption(read_via_option, "read-view"); + builder_options[read_via_option] = "read-view"; } ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, builder.Finish()); // Exercise the actual table/read merge before creating the dependency context. - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(builder_load_options, std::move(read_context), builder_options)); auto* append_only_table_read = dynamic_cast(table_read.get()); ASSERT_NE(append_only_table_read, nullptr); const auto& internal_context = append_only_table_read->context_; @@ -431,7 +539,9 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); - ASSERT_EQ(internal_context->GetTableSchema()->Options(), table_options); + // The operator uses the configured copy; the supplied schema stays unchanged. + ASSERT_EQ(internal_context->GetTableSchema()->Options(), expected_options); + ASSERT_EQ(table_schema->Options(), table_options); } } diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 435e2254f..4e4b8f062 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -25,6 +25,7 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/metrics.h" @@ -60,10 +61,14 @@ TEST(TableScanTest, TestDefaultMetricsSnapshot) { TEST(TableScanTest, TestNoSnapshot) { std::string path = paimon::test::GetDataDir() + "/orc/append_table_with_nested_type.db/append_table_with_nested_type/"; - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(builder_load_options, + std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_FALSE(plan->SnapshotId()); ASSERT_TRUE(plan->Splits().empty()); @@ -71,23 +76,31 @@ TEST(TableScanTest, TestNoSnapshot) { TEST(TableScanTest, TestNonExistTable) { std::string path = paimon::test::GetDataDir() + "/non-exist.db/non-exist/"; - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), "not found latest schema"); + ASSERT_NOK_WITH_MSG( + TableScan::Create(builder_load_options, std::move(context), builder_options), + "not found latest schema"); } TEST(TableScanTest, TestStreamingSnapshotsRemoved) { auto dir = UniqueTestDirectory::Create("local"); ASSERT_TRUE(dir); ASSERT_TRUE(TestUtil::CopyDirectory(GetDataDir() + "/orc/append_09.db/append_09", dir->Str())); - ScanContextBuilder builder(dir->Str()); - builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MODE, "latest") - .WithStreamingMode(true); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + builder_options[Options::SCAN_MODE] = "latest"; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(std::shared_ptr waiting_plan, table_scan->CreatePlan()); @@ -103,14 +116,19 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { std::string path = GetDataDir() + "/orc/append_09.db/append_09"; for (const std::string next_snapshot_id : {"6", "7"}) { SCOPED_TRACE(next_snapshot_id); - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .AddOption(Options::SCAN_SNAPSHOT_ID, next_snapshot_id) + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + builder_options[Options::SCAN_MODE] = "from-snapshot"; + builder_options[Options::SCAN_SNAPSHOT_ID] = next_snapshot_id; + builder + .WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); @@ -131,10 +149,14 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { TEST(TableScanTest, TestPkSchemaEvolutionScan) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table/"; - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(builder_load_options, + std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_TRUE(plan->SnapshotId()); ASSERT_FALSE(plan->Splits().empty()); @@ -164,14 +186,18 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; - ScanContextBuilder builder(path); - builder.AddOption(Options::FILE_FORMAT, "orc"); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::FILE_FORMAT] = "orc"; + builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), - "read-optimized system table does not support streaming scan for primary " - "key table"); + ASSERT_NOK_WITH_MSG( + TableScan::Create(builder_load_options, std::move(context), builder_options), + "read-optimized system table does not support streaming scan for primary " + "key table"); } } // namespace paimon::test diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 4ce86981d..d6da1bc37 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -396,18 +396,17 @@ Result> AuditLogSystemTable::ChangelogSchema( Result> AuditLogSystemTable::NewScan( std::unique_ptr ctx) const { + const auto& dynamic_options = options_; const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } std::shared_ptr scan_filter = context->GetScanFilters(); - ScanContextBuilder builder(table_path_); + ScanContextBuilder builder; builder.WithStreamingMode(context->IsStreamingMode()) - .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -423,20 +422,24 @@ Result> AuditLogSystemTable::NewScan( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, CreateDataTable(table_schema_, options_)); - return data_table->NewScan(std::move(data_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, + data_table->Copy(dynamic_options)); + return configured_table->NewScan(std::move(data_context)); } Result> AuditLogSystemTable::NewRead( std::unique_ptr ctx) const { + const auto& dynamic_options = options_; const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); - return NewChangelogRead(context, std::make_shared()); + return NewChangelogRead(context, std::make_shared(), dynamic_options); } Result> AuditLogSystemTable::NewChangelogRead( const std::shared_ptr& context, - std::shared_ptr converter) const { + std::shared_ptr converter, + const std::map& dynamic_options) const { if (table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented(Name(), " system table only supports primary key table"); } @@ -445,15 +448,16 @@ Result> AuditLogSystemTable::NewChangelogRead( } using StringMap = std::map; - PAIMON_ASSIGN_OR_RAISE(StringMap read_options, ReadOptions(context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(StringMap read_options, ReadOptions(dynamic_options)); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, CreateDataReadContext(context, read_options)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr data_table, - CreateDataTable(table_schema_, options_, context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options_)); + auto options = read_options; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, data_table->Copy(options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read, - data_table->NewRead(std::move(data_context))); + configured_table->NewRead(std::move(data_context))); auto* key_value_read = dynamic_cast(data_read.get()); if (!key_value_read) { return Status::Invalid("audit_log system table requires key-value table read"); @@ -473,16 +477,10 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema(core_options.TableReadSequenceNumberEnabled())); - auto options = read_options; - if (core_options.TableReadSequenceNumberEnabled()) { - options[Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED] = "true"; - } - ReadContextBuilder builder(table_path_); + ReadContextBuilder builder; builder.SetReadFieldNames(base_read_schema->field_names()) - .SetOptions(options) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(GetFileSystem()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) @@ -521,4 +519,14 @@ Result> AuditLogSystemTable::ReadOptions( return read_options; } +Result> AuditLogSystemTable::Copy( + const std::map& dynamic_options) const { + using OptionsMap = std::map; + PAIMON_ASSIGN_OR_RAISE(OptionsMap options, CopyOptions(options_, dynamic_options)); + auto table = std::make_shared(*this); + table->options_ = std::move(options); + PAIMON_RETURN_NOT_OK(table->AdoptOwnSchema()); + return std::shared_ptr
(std::move(table)); +} + } // namespace paimon diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 6331c53c0..192188b62 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -52,6 +52,9 @@ std::unique_ptr CreateChangelogBatchReader( /// System table for `T$audit_log`, exposing row-level changelog records with rowkind. class AuditLogSystemTable : public SystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "audit_log"; /// @param data_table_name Logical table name, including any branch suffix. @@ -71,7 +74,8 @@ class AuditLogSystemTable : public SystemTable { Result> NewChangelogRead( const std::shared_ptr& context, - std::shared_ptr converter) const; + std::shared_ptr converter, + const std::map& dynamic_options) const; /// Build the context for the data table underneath this changelog view. /// /// The builder starts from the defaults, so every setting the caller configured has to be diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 57c5a1ef3..316d6dbcc 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -97,10 +97,21 @@ Result> BinlogSystemTable::ChangelogSchema( Result> BinlogSystemTable::NewRead( std::unique_ptr ctx) const { + const auto& dynamic_options = options_; const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); - return NewChangelogRead(context, CreateBinlogBatchConverter()); + return NewChangelogRead(context, CreateBinlogBatchConverter(), dynamic_options); +} + +Result> BinlogSystemTable::Copy( + const std::map& dynamic_options) const { + using OptionsMap = std::map; + PAIMON_ASSIGN_OR_RAISE(OptionsMap options, CopyOptions(options_, dynamic_options)); + auto table = std::make_shared(*this); + table->options_ = std::move(options); + PAIMON_RETURN_NOT_OK(table->AdoptOwnSchema()); + return std::shared_ptr
(std::move(table)); } } // namespace paimon diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 8ec8bf192..eb7722054 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -34,6 +34,9 @@ std::shared_ptr CreateBinlogBatchConverter(); /// System table for `T$binlog`, exposing changelog records with list-wrapped data columns. class BinlogSystemTable : public AuditLogSystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "binlog"; /// @param data_table_name Logical table name, including any branch suffix. diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index b5f1c0442..06a4d8994 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -598,4 +598,28 @@ Result> PartitionsSystemTable::BuildRows() const { return rows; } +Result> CatalogOptionsSystemTable::Copy( + const std::map& /*dynamic_options*/) const { + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + +Result> AllTableOptionsSystemTable::Copy( + const std::map& /*dynamic_options*/) const { + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + +Result> TablesSystemTable::Copy( + const std::map& /*dynamic_options*/) const { + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + +Result> PartitionsSystemTable::Copy( + const std::map& /*dynamic_options*/) const { + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + } // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index 22d8a0262..e1fc1be95 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -46,6 +46,9 @@ struct GlobalSystemTableContext { /// as key/value rows. class CatalogOptionsSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "catalog_options"; static constexpr const char* kEnabledOption = "catalog-options-table.enabled"; @@ -62,6 +65,9 @@ class CatalogOptionsSystemTable : public InMemorySystemTable { /// all databases as (database_name, table_name, key, value) rows. class AllTableOptionsSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "all_table_options"; explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); @@ -77,6 +83,9 @@ class AllTableOptionsSystemTable : public InMemorySystemTable { /// databases including record counts and file statistics. class TablesSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "tables"; explicit TablesSystemTable(GlobalSystemTableContext context); @@ -92,6 +101,9 @@ class TablesSystemTable : public InMemorySystemTable { /// for all tables across all databases. class PartitionsSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "partitions"; explicit PartitionsSystemTable(GlobalSystemTableContext context); diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 5d5862d75..8520b29c9 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -117,7 +117,7 @@ Status InMemorySystemTable::CheckBranch(const std::map if (branch_ && branch != options.end() && BranchManager::NormalizeBranch(branch->second) != BranchManager::NormalizeBranch(*branch_)) { - return Status::Invalid("context branch contradicts the loaded table"); + return Status::Invalid("copy branch contradicts the loaded table"); } return Status::OK(); } @@ -125,7 +125,6 @@ Status InMemorySystemTable::CheckBranch(const std::map Result> InMemorySystemTable::NewScan( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - PAIMON_RETURN_NOT_OK(CheckBranch(context->GetOptions())); return std::make_unique(GetRootPath()); } @@ -134,9 +133,7 @@ Result> InMemorySystemTable::NewRead( const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); - PAIMON_RETURN_NOT_OK(CheckBranch(context->GetOptions())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - ForRead(context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, ForRead({})); return std::make_unique(std::move(table), context->GetMemoryPool()); } diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 5bd6fd5a0..52b74ba62 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -206,7 +206,7 @@ Result WithReadOptions( auto branch = overrides.find(Options::BRANCH); if (branch != overrides.end() && BranchManager::NormalizeBranch(branch->second) != context.branch) { - return Status::Invalid("context branch contradicts the loaded table"); + return Status::Invalid("copy branch contradicts the loaded table"); } MetadataSystemTableContext merged = context; for (const auto& [key, value] : overrides) { @@ -215,7 +215,7 @@ Result WithReadOptions( PAIMON_ASSIGN_OR_RAISE(TableType type, TableTypeDefine::FromOptions(merged.options)); PAIMON_ASSIGN_OR_RAISE(TableType bound_type, TableTypeDefine::FromOptions(context.options)); if (type != bound_type) { - return Status::Invalid("context type contradicts the loaded table"); + return Status::Invalid("copy type contradicts the loaded table"); } merged.options[Options::BRANCH] = context.branch; PAIMON_RETURN_NOT_OK(SchemaValidation::RejectQueryAuth(*context.table_schema, merged.options, @@ -1004,4 +1004,66 @@ Result> FilesSystemTable::BuildRows() const { return rows; } +Result> OptionsSystemTable::Copy( + const std::map& dynamic_options) const { + PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + +Result> SnapshotsSystemTable::Copy( + const std::map& dynamic_options) const { + PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + +Result> SchemasSystemTable::Copy( + const std::map& dynamic_options) const { + PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + +Result> TagsSystemTable::Copy( + const std::map& dynamic_options) const { + PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + +Result> BranchesSystemTable::Copy( + const std::map& dynamic_options) const { + PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + +Result> ConsumersSystemTable::Copy( + const std::map& dynamic_options) const { + PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); + auto table = std::make_shared(*this); + return std::shared_ptr
(std::move(table)); +} + +Result> ManifestsSystemTable::Copy( + const std::map& dynamic_options) const { + PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(MetadataSystemTableContext context, + WithReadOptions(context_, dynamic_options)); + auto table = std::make_shared(*this); + table->context_ = std::move(context); + return std::shared_ptr
(std::move(table)); +} + +Result> FilesSystemTable::Copy( + const std::map& dynamic_options) const { + PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(MetadataSystemTableContext context, + WithReadOptions(context_, dynamic_options)); + auto table = std::make_shared(*this); + table->context_ = std::move(context); + return std::shared_ptr
(std::move(table)); +} + } // namespace paimon diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index cfb8c6faf..a44c3ff72 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -33,6 +33,9 @@ class TableSchema; /// System table for `T$options`, exposing the latest base table options as key/value rows. class OptionsSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "options"; OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, @@ -57,6 +60,9 @@ struct MetadataSystemTableContext { /// System table for `T$snapshots`, exposing snapshot commit history. class SnapshotsSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "snapshots"; SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, @@ -72,6 +78,9 @@ class SnapshotsSystemTable : public InMemorySystemTable { /// System table for `T$schemas`, exposing schema evolution history. class SchemasSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "schemas"; SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); @@ -86,6 +95,9 @@ class SchemasSystemTable : public InMemorySystemTable { /// System table for `T$tags`, exposing tags and the snapshots they reference. class TagsSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "tags"; TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); @@ -100,6 +112,9 @@ class TagsSystemTable : public InMemorySystemTable { /// System table for `T$branches`, exposing table branches including `main`. class BranchesSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "branches"; BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); @@ -114,6 +129,9 @@ class BranchesSystemTable : public InMemorySystemTable { /// System table for `T$consumers`, exposing persisted streaming consumer offsets. class ConsumersSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "consumers"; ConsumersSystemTable(std::shared_ptr fs, std::string table_path, @@ -129,6 +147,9 @@ class ConsumersSystemTable : public InMemorySystemTable { /// System table for `T$manifests`, exposing data manifest metadata in the latest snapshot. class ManifestsSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "manifests"; ManifestsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, @@ -148,6 +169,9 @@ class ManifestsSystemTable : public InMemorySystemTable { /// System table for `T$files`, exposing data file metadata in the latest snapshot. class FilesSystemTable : public InMemorySystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "files"; FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 170e85c0d..90d28303c 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -59,6 +59,7 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption Result> ReadOptimizedSystemTable::NewScan( std::unique_ptr ctx) const { + const auto& dynamic_options = options_; const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { @@ -66,14 +67,10 @@ Result> ReadOptimizedSystemTable::NewScan( "PK real-time union read does not support read-optimized scans"); } auto options = ReadOptimizedOptions(); - auto read_options = context->GetOptions(); - read_options[kReadOptimizedScanOption] = "true"; - ScanContextBuilder builder(table_path_); + ScanContextBuilder builder; builder.WithStreamingMode(context->IsStreamingMode()) - .SetOptions(read_options) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(GetFileSystem()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -91,12 +88,15 @@ Result> ReadOptimizedSystemTable::NewScan( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, CreateDataTable(table_schema_, options)); - return data_table->NewScan(std::move(base_context)); + auto scan_options = dynamic_options; + scan_options[kReadOptimizedScanOption] = "true"; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, data_table->Copy(scan_options)); + return configured_table->NewScan(std::move(base_context)); } Result> ReadOptimizedSystemTable::CreateDataReadContext( const std::shared_ptr& context) const { - ReadContextBuilder builder(table_path_); + ReadContextBuilder builder; builder.SetPredicate(context->GetPredicate()) .EnablePredicateFilter(context->EnablePredicateFilter()) .EnablePrefetch(context->EnablePrefetch()) @@ -104,10 +104,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) .EnableMultiThreadRowToBatch(context->EnableMultiThreadRowToBatch()) .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) - .SetOptions(context->GetOptions()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(GetFileSystem()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) .SetWarmupLevel(context->GetWarmupLevel()) @@ -126,15 +124,27 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon Result> ReadOptimizedSystemTable::NewRead( std::unique_ptr ctx) const { + const auto& dynamic_options = options_; const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr data_table, - CreateDataTable(table_schema_, options_, context->GetFileSystemSchemeToIdentifierMap())); - return data_table->NewRead(std::move(base_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, + CreateDataTable(table_schema_, options_)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, + data_table->Copy(dynamic_options)); + return configured_table->NewRead(std::move(base_context)); +} + +Result> ReadOptimizedSystemTable::Copy( + const std::map& dynamic_options) const { + using OptionsMap = std::map; + PAIMON_ASSIGN_OR_RAISE(OptionsMap options, CopyOptions(options_, dynamic_options)); + auto table = std::make_shared(*this); + table->options_ = std::move(options); + PAIMON_RETURN_NOT_OK(table->AdoptOwnSchema()); + return std::shared_ptr
(std::move(table)); } } // namespace paimon diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 164b2c79c..5513585ab 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -32,6 +32,9 @@ class TableSchema; /// System table for `T$ro`, exposing read-optimized data. class ReadOptimizedSystemTable : public SystemTable { public: + Result> Copy( + const std::map& dynamic_options) const override; + static constexpr const char* kName = "ro"; /// @param data_table_name Logical table name, including any branch suffix. diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index f7542b8f8..aaa1029f5 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -237,18 +237,36 @@ Status SystemTable::AdoptOwnSchema() { Result> SystemTable::CreateDataTable( const std::shared_ptr& table_schema, - const std::map& options, - const std::map& fs_scheme_map_overrides) const { - auto fs_scheme_map = fs_scheme_map_; - for (const auto& [scheme, identifier] : fs_scheme_map_overrides) { - fs_scheme_map[scheme] = identifier; - } + const std::map& options) const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, table_schema->MergeOptions(options)); const std::string branch = BranchManager::NormalizeBranch(LoadBranch(merged_schema->Options())); return FileStoreTable::Create( GetFileSystem(), GetRootPath(), std::shared_ptr(std::move(merged_schema)), - CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", branch, fs_scheme_map); + CatalogUtils::IdentifierFromPath(GetRootPath(), branch), "", branch, fs_scheme_map_); +} + +Result> SystemTable::CopyOptions( + const std::map& bound_options, + const std::map& overrides) { + auto options = bound_options; + for (const auto& [key, value] : overrides) { + options[key] = value; + } + PAIMON_ASSIGN_OR_RAISE(std::string bound_branch, + OptionsUtils::GetValueFromMap( + bound_options, Options::BRANCH, BranchManager::DEFAULT_MAIN_BRANCH)); + auto branch = overrides.find(Options::BRANCH); + if (branch != overrides.end() && BranchManager::NormalizeBranch(branch->second) != + BranchManager::NormalizeBranch(bound_branch)) { + return Status::Invalid("copy branch contradicts the loaded table"); + } + PAIMON_ASSIGN_OR_RAISE(TableType bound_type, TableTypeDefine::FromOptions(bound_options)); + PAIMON_ASSIGN_OR_RAISE(TableType type, TableTypeDefine::FromOptions(options)); + if (type != bound_type) { + return Status::Invalid("copy type contradicts the loaded table"); + } + return options; } Result> SystemTable::NewWrite( diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 5ed41c7fd..65dc1a9a0 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -90,11 +90,15 @@ class SystemTable : public Table, public std::enable_shared_from_this> CopyOptions( + const std::map& bound_options, + const std::map& overrides); + /// Builds the data table using this view's data schema and options. Result> CreateDataTable( const std::shared_ptr& table_schema, - const std::map& options, - const std::map& fs_scheme_map_overrides = {}) const; + const std::map& options) const; friend class SystemTableLoader; friend class GlobalSystemTableLoader; diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index b1c2eadb9..1e1565b08 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -31,6 +31,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/generic_row.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" @@ -253,8 +254,8 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { - ReadContextBuilder builder("/tmp/table"); - builder.SetOptions(options).SetWarmupLevel(level); + ReadContextBuilder builder; + builder.SetWarmupLevel(level); ASSERT_OK_AND_ASSIGN(std::unique_ptr caller_unique_context, builder.Finish()); std::shared_ptr caller_context(std::move(caller_unique_context)); ASSERT_EQ(level, caller_context->GetWarmupLevel()); @@ -267,14 +268,13 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { // ChangelogTableRead that wraps it is local to audit_log_system_table.cpp, so a test cannot // name the type to reach the read underneath it. EXPECT rather than ASSERT, because the // three chains are independent and one broken chain must not hide another. - ASSERT_OK_AND_ASSIGN(auto audit_log_options, - audit_log.ReadOptions(caller_context->GetOptions())); + ASSERT_OK_AND_ASSIGN(auto audit_log_options, audit_log.ReadOptions({})); ASSERT_OK_AND_ASSIGN(std::unique_ptr audit_log_context, audit_log.CreateDataReadContext(caller_context, audit_log_options)); EXPECT_EQ(level, audit_log_context->GetWarmupLevel()) << "$audit_log dropped the caller's WarmupLevel"; - ASSERT_OK_AND_ASSIGN(auto binlog_options, binlog.ReadOptions(caller_context->GetOptions())); + ASSERT_OK_AND_ASSIGN(auto binlog_options, binlog.ReadOptions({})); ASSERT_OK_AND_ASSIGN(std::unique_ptr binlog_context, binlog.CreateDataReadContext(caller_context, binlog_options)); EXPECT_EQ(level, binlog_context->GetWarmupLevel()) diff --git a/src/paimon/core/table/table_factory.cpp b/src/paimon/core/table/table_factory.cpp index b9a2bc6b1..cdf92d08f 100644 --- a/src/paimon/core/table/table_factory.cpp +++ b/src/paimon/core/table/table_factory.cpp @@ -28,7 +28,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/table.h" #include "paimon/commit_context.h" -#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/path_util.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" @@ -51,347 +51,281 @@ namespace paimon { namespace { -Result> ManagedTable( - const std::string& path, const std::string& branch, const std::shared_ptr& schema, - CoreOptions options, const std::optional& identifier, - const std::optional& table_id, - const std::map& fs_scheme_map, - const std::shared_ptr& schema_manager = nullptr) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr table, - FileStoreTable::Create(std::make_unique(path, std::move(options)), schema, - identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)), - table_id.value_or(""), fs_scheme_map, schema_manager)); - return std::shared_ptr
(std::move(table)); + +Status CheckCatalogTable(const TableLoadOptions& load, const std::shared_ptr
& table) { + if (load.table_id && load.table_id != table->CatalogUuid()) { + return Status::Invalid("the expected table id does not match the catalog table id"); + } + if (!load.path.empty() && !table->GetRootPath().empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string supplied_path, PathUtil::NormalizePath(load.path)); + PAIMON_ASSIGN_OR_RAISE(std::string catalog_path, + PathUtil::NormalizePath(table->GetRootPath())); + if (supplied_path != catalog_path) { + return Status::Invalid("the table path does not match the catalog table location"); + } + } + if (std::dynamic_pointer_cast(table) == nullptr && + (load.table_schema_json || load.file_system || !load.fs_scheme_to_identifier_map.empty())) { + return Status::Invalid( + "a catalog format or system table carries its schema and file system"); + } + return Status::OK(); } -} // namespace -Result> TableFactory::Create(const WriteContext* ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); +} // namespace - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return std::shared_ptr
(given_table); +Result> TableFactory::Create( + const TableLoadOptions& load, const std::map& overrides, + const char* operation) { + if (load.catalog == nullptr && load.identifier) { + return Status::Invalid("a table identifier requires a non-null catalog"); } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); + if (load.catalog != nullptr && !load.identifier) { + return Status::Invalid("a catalog load requires a table identifier"); + } + if (load.file_system && !load.fs_scheme_to_identifier_map.empty()) { + return Status::Invalid("a file system and URI scheme routing cannot be specified together"); + } + auto fs_scheme_map = load.fs_scheme_to_identifier_map; + const bool writing = std::string(operation) == "write" || std::string(operation) == "commit"; + if (writing && load.identifier) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(*load.identifier, operation)); + } + std::optional named_system_path; + if (!load.path.empty()) { + PAIMON_ASSIGN_OR_RAISE(named_system_path, SystemTableLoader::TryParsePath(load.path)); + } + PAIMON_ASSIGN_OR_RAISE( + std::string branch, + BranchManager::ResolveBranch(load.identifier, overrides, + named_system_path ? named_system_path->branch : std::nullopt, + operation)); + std::optional identifier = load.identifier; + if (load.catalog) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); + PAIMON_ASSIGN_OR_RAISE(Identifier resolved_identifier, + CatalogUtils::ResolvedBranchIdentifier(*identifier, branch)); + identifier.emplace(std::move(resolved_identifier)); + } + auto dynamic_options = overrides; + if (load.identifier || dynamic_options.count(Options::BRANCH) || + (named_system_path && named_system_path->branch)) { + dynamic_options[Options::BRANCH] = branch; + } + if (writing && load.catalog) { + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(load.catalog)); + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(branch)); + PAIMON_ASSIGN_OR_RAISE(std::optional named_branch, + load.identifier->GetBranchName()); + if (!BranchManager::IsMainBranch(branch) && !named_branch) { + return Status::Invalid( + "a catalog write or commit must name its branch in the table identifier"); } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); } - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); + std::string path = load.path; + std::optional table_id = load.table_id; + std::optional schema_json = load.table_schema_json; + std::shared_ptr file_system = load.file_system; + // A catalog-only load obtains identity, schema and location from one table generation. + if (load.catalog && (path.empty() || load.table_id)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, load.catalog->GetTable(*identifier)); + PAIMON_RETURN_NOT_OK(CheckCatalogTable(load, table)); + if (std::dynamic_pointer_cast(table) == nullptr) { + return table; } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); + if (path.empty()) { + path = table->GetRootPath(); + if (!schema_json) { + PAIMON_ASSIGN_OR_RAISE(schema_json, table->GetSchema()->GetJsonSchema()); + } } + if (!file_system && fs_scheme_map.empty()) { + file_system = table->GetFileSystem(); + } + if (auto managed = std::dynamic_pointer_cast(table)) { + auto inherited_map = managed->GetFileSystemSchemeToIdentifierMap(); + for (const auto& [scheme, identifier] : fs_scheme_map) { + inherited_map[scheme] = identifier; + } + fs_scheme_map = std::move(inherited_map); + } + table_id = table->CatalogUuid(); } - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(format_table); - } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + PAIMON_ASSIGN_OR_RAISE(path, PathUtil::NormalizePath(path)); + if (path.empty()) { + return Status::Invalid("table path is empty"); } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; + if (load.catalog && !file_system && fs_scheme_map.empty()) { + PAIMON_ASSIGN_OR_RAISE(file_system, load.catalog->GetTableFileSystem(*identifier)); } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); - return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options), - ctx->GetIdentifier(), std::nullopt, - ctx->GetFileSystemSchemeToIdentifierMap(), schema_manager); -} - -Result> TableFactory::Create(const CommitContext* ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return std::shared_ptr
(given_table); + if (writing && load.catalog && BranchManager::IsMainBranch(branch) && !schema_json) { + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, + CatalogUtils::BranchIdentifier(*load.identifier, branch)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + load.catalog->LoadTableSchema(identifier)); + PAIMON_ASSIGN_OR_RAISE(schema_json, schema->GetJsonSchema()); } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); + PAIMON_ASSIGN_OR_RAISE(CoreOptions temporary, + CoreOptions::FromMap(dynamic_options, file_system, fs_scheme_map)); + PAIMON_ASSIGN_OR_RAISE(std::optional system_path, + SystemTableLoader::TryParsePath(path)); + if (system_path) { + if (writing) { + return Status::Invalid("system tables are read-only"); + } + if (!writing) { + PAIMON_ASSIGN_OR_RAISE(dynamic_options, CatalogUtils::ResolveQueryAuthReadOptions( + load.catalog, identifier, dynamic_options, + branch, temporary.GetFileSystem(), path, + schema_json.has_value(), operation)); } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); + std::shared_ptr table, + SystemTableLoader::LoadFromPath(temporary.GetFileSystem(), path, dynamic_options, + fs_scheme_map, schema_json)); + return std::shared_ptr
(std::move(table)); + } + std::shared_ptr schema; + const std::string metadata_branch = branch; + auto schema_manager = std::make_shared(temporary.GetFileSystem(), path, branch); + Result> format_result = + FormatTableLoader::TryLoad(temporary.GetFileSystem(), path, branch, dynamic_options, + schema_json, schema_manager.get(), &schema); + if (load.catalog && BranchManager::IsMainBranch(branch) && + (!format_result.ok() || format_result.value() != nullptr || !schema)) { + // An external format table may have real data under `schema`, including file names + // that look like schema versions. Its catalog determines the directory semantics. + Result> catalog_result = load.catalog->GetTable(*load.identifier); + if (catalog_result.ok() && + std::dynamic_pointer_cast(catalog_result.value()) != nullptr) { + PAIMON_RETURN_NOT_OK(CheckCatalogTable(load, catalog_result.value())); + return catalog_result; } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); + PAIMON_RETURN_NOT_OK(format_result.status()); + // Preserve catalog lookup failures (including a missing upstream table) when there + // is no usable schema at the supplied location. + PAIMON_RETURN_NOT_OK(catalog_result.status()); + if (format_result.value() != nullptr) { + return Status::Invalid("the catalog did not return a format table"); } } - - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(format_table); - } - if (latest_schema == nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, std::move(format_result)); + if (format_table) { + if (load.table_id) { + return Status::Invalid("a format table has no catalog commit table id"); + } + return std::shared_ptr
(std::move(format_table)); + } + if (!schema) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); - return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options), - ctx->GetIdentifier(), ctx->GetTableId(), {}, schema_manager); -} - -Result> TableFactory::Create(const ScanContext* context) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return std::shared_ptr
(format_table); + if (!writing) { + PAIMON_ASSIGN_OR_RAISE(dynamic_options, CatalogUtils::ResolveQueryAuthReadOptions( + load.catalog, identifier, dynamic_options, + branch, temporary.GetFileSystem(), path, + schema_json.has_value(), operation)); } - // load schema - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - if (context->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(tmp_options.GetBranch())); + auto merged = schema->Options(); + for (const auto& [key, value] : dynamic_options) { + merged[key] = value; } - std::map read_options; - PAIMON_ASSIGN_OR_RAISE( - read_options, - CatalogUtils::ResolveQueryAuthReadOptions( - context->GetCatalog(), context->GetIdentifier(), context->GetOptions(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), tmp_options.GetFileSystem(), - context->GetPath(), context->GetSpecificTableSchema().has_value(), "scan")); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { + if (!writing) { PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), - read_options, {}, context->GetSpecificTableSchema())); - return std::shared_ptr
(system_table); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - read_options, context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(format_table); + branch, BranchManager::ResolveBranch(load.identifier, merged, std::nullopt, operation)); } - if (!latest_schema) { - return Status::Invalid("not found latest schema"); + if (writing || merged.count(Options::BRANCH) != 0) { + merged[Options::BRANCH] = branch; } - auto options_map = latest_schema->Options(); - for (const auto& [key, value] : read_options) { - options_map[key] = value; - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(std::nullopt, options_map, std::nullopt, "scan")); - if (options_map.count(Options::BRANCH) != 0) { - options_map[Options::BRANCH] = branch; + if (metadata_branch != branch) { + // A supplied main schema may select another branch for reads. Its metadata manager + // must not become the schema cache of that other branch. + schema_manager.reset(); } PAIMON_ASSIGN_OR_RAISE(CoreOptions options, - CoreOptions::FromMap(options_map, context->GetSpecificFileSystem(), {})); - return ManagedTable(context->GetPath(), branch, latest_schema, std::move(options), - context->GetIdentifier(), std::nullopt, {}); + CoreOptions::FromMap(merged, file_system, fs_scheme_map)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(std::make_unique(path, std::move(options)), schema, + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)), + table_id.value_or(""), fs_scheme_map, load.catalog, dynamic_options, + schema_manager)); + table->supplied_main_schema_ = load.table_schema_json; + table->fallback_file_system_ = file_system; + return std::shared_ptr
(std::move(table)); } -Result> TableFactory::Create(const ReadContext* context) { +Result> TableRead::Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return std::shared_ptr
(format_table); - } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - std::map read_options; - PAIMON_ASSIGN_OR_RAISE( - read_options, - CatalogUtils::ResolveQueryAuthReadOptions( - context->GetCatalog(), context->GetIdentifier(), context->GetOptions(), - context->GetBranch(), tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetSpecificTableSchema().has_value(), "read")); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - if (context->EnablePreciseFileIndexEvaluate()) { - return Status::NotImplemented( - "Precise file-index evaluation is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath( - tmp_core_options.GetFileSystem(), context->GetPath(), read_options, - context->GetFileSystemSchemeToIdentifierMap(), context->GetSpecificTableSchema())); - return std::shared_ptr
(system_table); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), read_options, - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(format_table); + if (load_options.table_id && !load_options.catalog) { + return Status::Invalid("a table id requires a catalog or an external REST commit request"); } - - if (!latest_schema) { - return Status::Invalid("not found latest schema"); - } - auto options_map = latest_schema->Options(); - for (const auto& [key, value] : read_options) { - options_map[key] = value; - } - PAIMON_ASSIGN_OR_RAISE( - std::string branch, - BranchManager::ResolveBranch(std::nullopt, options_map, std::nullopt, "read")); - if (options_map.count(Options::BRANCH) != 0) { - options_map[Options::BRANCH] = branch; + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::string branch, + BranchManager::ResolveBranch(load_options.identifier, + dynamic_options, std::nullopt, "read")); + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); } - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, - CoreOptions::FromMap(options_map, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, - ManagedTable(context->GetPath(), branch, latest_schema, - std::move(options), context->GetIdentifier(), std::nullopt, - context->GetFileSystemSchemeToIdentifierMap())); - checked_pointer_cast(table)->loaded_from_read_context_ = true; - return table; + TableFactory::Create(load_options, dynamic_options, "read")); + PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); + return table->NewRead(std::move(context)); } -Result> TableScan::Create(std::unique_ptr context) { +Result> TableScan::Create( + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + if (load_options.table_id && !load_options.catalog) { + return Status::Invalid("a table id requires a catalog or an external REST commit request"); + } + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::string branch, + BranchManager::ResolveBranch(load_options.identifier, + dynamic_options, std::nullopt, "scan")); + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + TableFactory::Create(load_options, dynamic_options, "scan")); + PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); return table->NewScan(std::move(context)); } -Result> TableRead::Create(std::unique_ptr context) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); - return table->NewRead(std::move(context)); -} - Result> FileStoreWrite::Create( - std::unique_ptr context) { + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + if (load_options.table_id && !load_options.catalog) { + return Status::Invalid("a table id requires a catalog or an external REST commit request"); + } + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + load_options.identifier, dynamic_options, + std::nullopt, "write")); + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + TableFactory::Create(load_options, dynamic_options, "write")); + PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); return table->NewWrite(std::move(context)); } Result> FileStoreCommit::Create( - std::unique_ptr context) { + const TableLoadOptions& load_options, std::unique_ptr context, + const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(context.get())); + if (load_options.catalog && context->UseRESTCatalogCommit()) { + return Status::Invalid( + "a commit cannot both use a catalog and build an external REST request"); + } + if (load_options.table_id && !load_options.catalog && !context->UseRESTCatalogCommit()) { + return Status::Invalid("a table id requires a catalog or an external REST commit request"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + TableFactory::Create(load_options, dynamic_options, "commit")); + PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); return table->NewCommit(std::move(context)); } + } // namespace paimon diff --git a/src/paimon/core/table/table_factory.h b/src/paimon/core/table/table_factory.h index 34157f93f..564a6baca 100644 --- a/src/paimon/core/table/table_factory.h +++ b/src/paimon/core/table/table_factory.h @@ -17,24 +17,25 @@ */ #pragma once + +#include #include +#include +#include "paimon/catalog/table_load_options.h" #include "paimon/result.h" + namespace paimon { class Table; -class ScanContext; -class ReadContext; -class WriteContext; -class CommitContext; -/// Compatibility loading for the existing static operator factories. + +/// Loads the table used by a static operator factory. class TableFactory { public: TableFactory() = delete; ~TableFactory() = delete; - static Result> Create(const ScanContext* context); - static Result> Create(const ReadContext* context); - static Result> Create(const WriteContext* context); - static Result> Create(const CommitContext* context); + static Result> Create( + const TableLoadOptions& load_options, + const std::map& dynamic_options, const char* operation); }; } // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 78b14adc0..c97752a5e 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -34,6 +34,7 @@ #include "arrow/ipc/json_simple.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/catalog/catalog_context.h" #include "paimon/common/fs/resolving_file_system.h" @@ -86,6 +87,7 @@ #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/utils/deletion_vector_test_helper.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" @@ -120,6 +122,58 @@ Result> CreateManagedTable(const std::shared_ptrGetFileSystem(); + const std::string path = dir->Str() + "/db.db/tbl"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
created, + CreateManagedTable(fs, path, Identifier("db", "tbl"))); + auto catalog = + std::make_shared(fs, dir->Str(), std::map{}); + std::weak_ptr weak_catalog = catalog; + const std::map routing = {{"file", "local"}}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, path, checked_pointer_cast(created->GetSchema()), + Identifier("db", "tbl"), "catalog-uuid", "main", routing, catalog)); + const auto original_options = checked_pointer_cast(table->GetSchema())->Options(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
first, + table->Copy({{Options::TARGET_FILE_ROW_NUM, "1"}, {Options::FILE_FORMAT, "parquet"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
second, + first->Copy({{Options::TARGET_FILE_ROW_NUM, "2"}})); + ASSERT_EQ(original_options, checked_pointer_cast(table->GetSchema())->Options()); + ASSERT_EQ("1", checked_pointer_cast(first->GetSchema()) + ->Options() + .at(Options::TARGET_FILE_ROW_NUM)); + ASSERT_EQ("2", checked_pointer_cast(second->GetSchema()) + ->Options() + .at(Options::TARGET_FILE_ROW_NUM)); + ASSERT_EQ( + "parquet", + checked_pointer_cast(second->GetSchema())->Options().at(Options::FILE_FORMAT)); + ASSERT_EQ(table->GetRootPath(), second->GetRootPath()); + ASSERT_EQ(table->FullName(), second->FullName()); + ASSERT_EQ(table->CatalogUuid(), second->CatalogUuid()); + ASSERT_EQ(fs, second->GetFileSystem()); + ASSERT_EQ(routing, + checked_pointer_cast(second)->GetFileSystemSchemeToIdentifierMap()); + ASSERT_NOK_WITH_MSG(second->Copy({{Options::BRANCH, "dev"}}), "branch contradicts"); + ASSERT_NOK_WITH_MSG(second->Copy({{Options::TYPE, "format-table"}}), "type contradicts"); + catalog.reset(); + table.reset(); + first.reset(); + ASSERT_FALSE(weak_catalog.expired()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, second->NewScan(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_TRUE(plan->Splits().empty()); + scan.reset(); + second.reset(); + ASSERT_TRUE(weak_catalog.expired()); +} + TEST(TableTest, TestGetTableWithUnknownDatabase) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -230,6 +284,47 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { } } +TEST(TableTest, TestScanWithMainBranchAlias) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string table_path = dir->Str() + "/db.db/tbl"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, + CreateManagedTable(fs, table_path, Identifier("db", "tbl"))); + auto catalog = + std::make_shared(fs, dir->Str(), std::map()); + for (const std::string alias : {"main", "MAIN", "Main", "mAiN"}) { + SCOPED_TRACE(alias); + const Identifier identifier("db", "tbl$branch_" + alias); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(identifier)); + for (bool explicit_main : {false, true}) { + SCOPED_TRACE(explicit_main); + TableLoadOptions load_options{table_path}; + load_options.catalog = catalog; + load_options.identifier.emplace(identifier); + std::map dynamic_options; + if (explicit_main) { + dynamic_options[Options::BRANCH] = "main"; + } + ScanContextBuilder builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr static_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr static_scan, + TableScan::Create(load_options, std::move(static_context), dynamic_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr static_plan, static_scan->CreatePlan()); + ASSERT_TRUE(static_plan->Splits().empty()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr loaded_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, + table->Copy(dynamic_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr loaded_scan, + configured_table_1->NewScan(std::move(loaded_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr loaded_plan, loaded_scan->CreatePlan()); + ASSERT_TRUE(loaded_plan->Splits().empty()); + } + } +} + TEST(TableTest, TestCurrentBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -301,12 +396,18 @@ TEST(TableTest, TestGetTableDispatchesByTableType) { ASSERT_EQ((std::vector{"key", "value"}), system_table->GetSchema()->FieldNames()); ASSERT_EQ(nullptr, std::dynamic_pointer_cast(system_table->GetSchema())); for (bool direct : {false, true}) { + TableLoadOptions context_builder_load_options{managed_dir + "$options"}; + std::map context_builder_options; + ReadContextBuilder context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(managed_dir + "$options") - .EnablePreciseFileIndexEvaluate(true) - .Finish()); - auto read = direct ? system_table->NewRead(std::move(context)) - : TableRead::Create(std::move(context)); + context_builder.EnablePreciseFileIndexEvaluate(true).Finish()); + auto read = direct ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + system_table->Copy(context_builder_options)); + return configured->NewRead(std::move(context)); + })() + : TableRead::Create(context_builder_load_options, std::move(context), + context_builder_options); ASSERT_NOK_WITH_MSG(read, "Precise file-index evaluation is not supported for system tables"); } @@ -362,10 +463,13 @@ TEST(TableTest, TestNewScanCreatesATableScan) { catalog->SetTableSchema(table->GetSchema()); catalog->SetTableLocation(dir->Str()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, - ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + std::map ctx_builder_options; + ScanContextBuilder ctx_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_3, + table->Copy(ctx_builder_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - table->NewScan(std::move(ctx))); + configured_table_3->NewScan(std::move(ctx))); } TEST(TableTest, TestNewReadCreatesATableRead) { @@ -380,16 +484,31 @@ TEST(TableTest, TestNewReadCreatesATableRead) { catalog->SetTableSchema(table->GetSchema()); catalog->SetTableLocation(dir->Str()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, - ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + std::map ctx_builder_18211_options; + ReadContextBuilder ctx_builder_18211; + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder_18211.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_4, + table->Copy(ctx_builder_18211_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - table->NewRead(std::move(ctx))); + configured_table_4->NewRead(std::move(ctx))); } namespace { class MockQueryAuthCatalog : public MockVersionManagedCatalog, public QueryAuthCatalog { public: + Result> GetTable(const Identifier& identifier) const override { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, GetTableFileSystem(identifier)); + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(fs, table_location, checked_pointer_cast(schema), + identifier, "", branch, {}, + std::const_pointer_cast(shared_from_this()))); + return std::shared_ptr
(std::move(table)); + } + Result> AuthTableQuery( const Identifier& identifier, const std::optional>& select) const override { @@ -513,8 +632,11 @@ std::string CompoundFilter(const std::string& function, const std::string& left, } Result> ReadAll( - std::unique_ptr context, const std::vector>& splits) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, TableRead::Create(std::move(context))); + const TableLoadOptions& load, std::unique_ptr context, + const std::vector>& splits, + const std::map& options = {}) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + TableRead::Create(load, std::move(context), options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr chunks, ReadResultCollector::CollectResult(std::move(reader))); @@ -539,8 +661,11 @@ Result> PlanQueryAuth(const std::string& path, const std::shared_ptr& catalog, const std::shared_ptr& predicate, std::optional limit) { - ScanContextBuilder builder(path); - builder.WithCatalog(catalog, Identifier("foo", "bar")); + TableLoadOptions builder_load{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_load.catalog = catalog; + builder_load.identifier.emplace(Identifier("foo", "bar")); if (predicate != nullptr) { builder.SetPredicate(predicate); } @@ -548,26 +673,34 @@ Result> PlanQueryAuth(const std::string& path, builder.SetLimit(limit.value()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, TableScan::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + TableScan::Create(builder_load, std::move(context), builder_options)); return scan->CreatePlan(); } Result> PlanAndReadAll( const std::string& path, const std::shared_ptr& catalog, const std::vector& read_fields = {}) { - ScanContextBuilder scan_builder(path); - scan_builder.WithCatalog(catalog, Identifier("foo", "bar")); - ReadContextBuilder read_builder(path); + TableLoadOptions scan_builder_load{path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_load.catalog = catalog; + scan_builder_load.identifier.emplace(Identifier("foo", "bar")); + TableLoadOptions read_builder_load{path}; + std::map read_builder_options; + ReadContextBuilder read_builder; if (!read_fields.empty()) { scan_builder.SetReadFieldNames(read_fields); read_builder.SetReadFieldNames(read_fields); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + TableScan::Create(scan_builder_load, std::move(scan_context), scan_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); - return ReadAll(std::move(read_context), plan->Splits()); + return ReadAll(read_builder_load, std::move(read_context), plan->Splits(), + read_builder_options); } } // namespace @@ -578,21 +711,17 @@ TEST(TableTest, TestLoadedTableAppliesQueryAuthThroughVirtualOperators) { ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, CreateQueryAuthTable(dir->Str())); ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); - FileSystemCatalog file_catalog(dir->GetFileSystem(), dir->Str(), {}); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - file_catalog.GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); catalog->filter = {kSalaryAbove150}; catalog->column_masking = {{"phone", kPhonePrefixMask}}; - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(table->GetRootPath()) - .WithCatalog(catalog, Identifier("foo", "bar")) - .Finish()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); ASSERT_NE(nullptr, std::dynamic_pointer_cast(plan->Splits()[0])); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(table->GetRootPath()).Finish()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); ASSERT_NOK_WITH_MSG( read->CreateReader( @@ -621,10 +750,13 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { R"({"kind": "LEAF", "transform": {"name": "FIELD_REF", "fieldRef": {"index": 0,)" R"( "name": "id", "type": "INT NOT NULL"}}, "function": "GREATER_THAN", "literals": [1]})"}; - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path).WithCatalog(catalog, identifier).Finish()); + TableLoadOptions scan_context_load_6{path}; + + scan_context_load_6.catalog = catalog; + scan_context_load_6.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_6, std::move(scan_context), {})); ASSERT_TRUE(catalog->auth_identifiers.empty()); catalog->column_masking = {{"phone", kPhonePrefixMask}, {"salary", R"({"name": "NULL"})"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); @@ -636,33 +768,42 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { ASSERT_NE(nullptr, std::dynamic_pointer_cast(split)); } - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(path).Finish()); + TableLoadOptions read_context_load_7{path}; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, - ReadAll(std::move(read_context), plan->Splits())); + ReadAll(read_context_load_7, std::move(read_context), plan->Splits(), {})); AssertColumn(*rows, "id", "[2, 3]"); AssertColumn(*rows, "name", R"(["bob", "carol"])"); AssertColumn(*rows, "phone", R"(["139", null])"); AssertColumn(*rows, "salary", "[null, null]"); - ASSERT_OK_AND_ASSIGN(read_context, - ReadContextBuilder(path).SetReadFieldNames({"name"}).Finish()); - ASSERT_OK_AND_ASSIGN(rows, ReadAll(std::move(read_context), plan->Splits())); + TableLoadOptions read_context_load_8{path}; + + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"name"}).Finish()); + ASSERT_OK_AND_ASSIGN(rows, + ReadAll(read_context_load_8, std::move(read_context), plan->Splits(), {})); AssertColumn(*rows, "name", R"(["bob", "carol"])"); ASSERT_EQ(nullptr, rows->GetFieldByName("id")); - ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder(path).SetReadFieldIds({1}).Finish()); - ASSERT_OK_AND_ASSIGN(rows, ReadAll(std::move(read_context), plan->Splits())); + TableLoadOptions read_context_load_9{path}; + + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldIds({1}).Finish()); + ASSERT_OK_AND_ASSIGN(rows, + ReadAll(read_context_load_9, std::move(read_context), plan->Splits(), {})); AssertColumn(*rows, "name", R"(["bob", "carol"])"); ASSERT_EQ(nullptr, rows->GetFieldByName("id")); - ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder(path) + TableLoadOptions read_context_load_10{path}; + + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder() .SetReadFieldNames({"name", "phone"}) .SetPredicate(PredicateBuilder::Equal( /*field_index=*/1, "phone", FieldType::STRING, Literal(FieldType::STRING, "139", 3))) .EnablePredicateFilter(true) .Finish()); - ASSERT_OK_AND_ASSIGN(rows, ReadAll(std::move(read_context), plan->Splits())); + ASSERT_OK_AND_ASSIGN( + rows, ReadAll(read_context_load_10, std::move(read_context), plan->Splits(), {})); AssertColumn(*rows, "name", R"(["bob"])"); AssertColumn(*rows, "phone", R"(["139"])"); ASSERT_EQ(nullptr, rows->GetFieldByName("id")); @@ -682,8 +823,11 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { ASSERT_EQ(original->GetScope(), auth_split->GetScope()); deserialized.push_back(copy); } - ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder(path).Finish()); - ASSERT_OK_AND_ASSIGN(rows, ReadAll(std::move(read_context), deserialized)); + TableLoadOptions read_context_load_11{path}; + + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(rows, + ReadAll(read_context_load_11, std::move(read_context), deserialized, {})); AssertColumn(*rows, "phone", R"(["139", null])"); const auto& planned = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -695,26 +839,33 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { ASSERT_OK_AND_ASSIGN( std::shared_ptr java_planned_split, Split::Deserialize(java_planned.data(), java_planned.size(), GetDefaultPool())); - ASSERT_OK_AND_ASSIGN(read_context, - ReadContextBuilder(path).SetReadFieldNames({"name"}).Finish()); - ASSERT_NOK_WITH_MSG(ReadAll(std::move(read_context), {java_planned_split}), - "The split does not record which columns its query auth rules were " - "authorized for"); + TableLoadOptions read_context_load_12{path}; + + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"name"}).Finish()); + ASSERT_NOK_WITH_MSG( + ReadAll(read_context_load_12, std::move(read_context), {java_planned_split}, {}), + "The split does not record which columns its query auth rules were " + "authorized for"); catalog->filter = {}; ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "Query auth rules changed"); ASSERT_EQ(2u, catalog->auth_identifiers.size()); - ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder(path) - .WithCatalog(catalog, identifier) - .SetPartitionFilter({{{"phone", "139"}}}) - .Finish()); - ASSERT_OK_AND_ASSIGN(scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_load_13{path}; + + scan_context_load_13.catalog = catalog; + scan_context_load_13.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, + ScanContextBuilder().SetPartitionFilter({{{"phone", "139"}}}).Finish()); + ASSERT_OK_AND_ASSIGN(scan, + TableScan::Create(scan_context_load_13, std::move(scan_context), {})); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "masked partition key 'phone'"); - ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder(path).Finish()); + TableLoadOptions read_context_load_14{path}; + + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_load_14, std::move(read_context), {})); ASSERT_NOK_WITH_MSG( read->CreateReader( std::dynamic_pointer_cast(plan->Splits()[0])->GetSplit()), @@ -741,17 +892,25 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { catalog->filter = { R"({"kind": "LEAF", "transform": {"name": "FIELD_REF", "fieldRef": {"index": 9,)" R"( "name": "gone", "type": "INT"}}, "function": "IS_NULL", "literals": []})"}; - ASSERT_OK_AND_ASSIGN(scan_context, - ScanContextBuilder(path).WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_load_15{path}; + + scan_context_load_15.catalog = catalog; + scan_context_load_15.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(scan, + TableScan::Create(scan_context_load_15, std::move(scan_context), {})); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "Query auth of Identifier{database='foo', object='bar'} (branch main, " "schema 0): Row filter references column 'gone' which does not exist"); catalog->filter = {}; catalog->column_masking = {{"name", R"({"name": "UPPER", "inputs": []})"}}; - ASSERT_OK_AND_ASSIGN(scan_context, - ScanContextBuilder(path).WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_load_16{path}; + + scan_context_load_16.catalog = catalog; + scan_context_load_16.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(scan, + TableScan::Create(scan_context_load_16, std::move(scan_context), {})); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "Column mask transform 'UPPER' is not supported"); catalog->column_masking = {}; for (const std::string literals : {"", R"(, "literals": null)"}) { @@ -759,9 +918,13 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { R"({{"kind": "LEAF", "transform": {{"name": "FIELD_REF", "fieldRef": {{"index": 0,)" R"( "name": "id", "type": "INT NOT NULL"}}}}, "function": "NOT_IN"{}}})", literals)}; - ASSERT_OK_AND_ASSIGN(scan_context, - ScanContextBuilder(path).WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_load_17{path}; + + scan_context_load_17.catalog = catalog; + scan_context_load_17.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(scan, + TableScan::Create(scan_context_load_17, std::move(scan_context), {})); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "Query auth row filter NOT_IN requires a non-null 'literals' array."); } @@ -780,50 +943,65 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); catalog->column_masking = {{"phone", R"({"name": "NULL"})"}}; + TableLoadOptions scan_context_load_18{path}; + + scan_context_load_18.catalog = catalog; + scan_context_load_18.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path) - .WithCatalog(catalog, identifier) - .SetReadFieldNames({"name"}) - .Finish()); + ScanContextBuilder().SetReadFieldNames({"name"}).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_18, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr name_plan, scan->CreatePlan()); ASSERT_FALSE(std::dynamic_pointer_cast(name_plan->Splits()[0]) ->GetAuthResult() ->HasRules()); catalog->column_masking = {}; - ASSERT_OK_AND_ASSIGN(scan_context, - ScanContextBuilder(path).WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_load_19{path}; + + scan_context_load_19.catalog = catalog; + scan_context_load_19.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(scan, + TableScan::Create(scan_context_load_19, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr all_plan, scan->CreatePlan()); catalog->column_masking = {{"phone", R"({"name": "NULL"})"}}; - ASSERT_OK_AND_ASSIGN(scan_context, - ScanContextBuilder(path).WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_load_20{path}; + + scan_context_load_20.catalog = catalog; + scan_context_load_20.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(scan, + TableScan::Create(scan_context_load_20, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr masked_plan, scan->CreatePlan()); catalog->column_masking = {}; catalog->filter = {CompoundFilter( "OR", ConstantFilter("TRUE"), R"({"kind": "LEAF", "transform": {"name": "FIELD_REF", "fieldRef": {"index": 2,)" R"( "name": "phone", "type": "STRING"}}, "function": "IS_NULL", "literals": []})")}; - ASSERT_OK_AND_ASSIGN(scan_context, - ScanContextBuilder(path).WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN(scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_load_21{path}; + + scan_context_load_21.catalog = catalog; + scan_context_load_21.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(scan, + TableScan::Create(scan_context_load_21, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr folded_plan, scan->CreatePlan()); catalog->filter = {}; catalog->column_masking = {{"phone", R"({"name": "NULL"})"}}; - ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder(path) - .WithCatalog(catalog, identifier) - .SetReadFieldNames({"name"}) - .Finish()); + TableLoadOptions scan_context_load_22{path}; + + scan_context_load_22.catalog = catalog; + scan_context_load_22.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().SetReadFieldNames({"name"}).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr unplanned_scan, - TableScan::Create(std::move(scan_context))); - ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder(path) - .WithCatalog(catalog, identifier) - .SetReadFieldNames({"id"}) - .Finish()); + TableScan::Create(scan_context_load_22, std::move(scan_context), {})); + TableLoadOptions scan_context_load_23{path}; + + scan_context_load_23.catalog = catalog; + scan_context_load_23.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().SetReadFieldNames({"id"}).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr id_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_23, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::string schema_json, table_schema.value()->ToJsonString()); rapidjson::Document document; @@ -858,29 +1036,43 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { "Query auth rules reference column 'secret', which the schema the scan " "planned with did not hold"); + TableLoadOptions read_context_load_24{path}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(path).SetReadFieldNames({"name"}).Finish()); - ASSERT_NOK_WITH_MSG(ReadAll(std::move(read_context), name_plan->Splits()), - "Column 'name' is read as field 2, but the scan authorized field 1 under " - "that name"); - ASSERT_OK_AND_ASSIGN(read_context, - ReadContextBuilder(path).SetReadFieldNames({"secret"}).Finish()); - ASSERT_NOK_WITH_MSG(ReadAll(std::move(read_context), all_plan->Splits()), - "Column 'secret' is read, but the scan did not ask the catalog to " - "authorize it"); + ReadContextBuilder().SetReadFieldNames({"name"}).Finish()); + ASSERT_NOK_WITH_MSG( + ReadAll(read_context_load_24, std::move(read_context), name_plan->Splits(), {}), + "Column 'name' is read as field 2, but the scan authorized field 1 under " + "that name"); + TableLoadOptions read_context_load_25{path}; + + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"secret"}).Finish()); + ASSERT_NOK_WITH_MSG( + ReadAll(read_context_load_25, std::move(read_context), all_plan->Splits(), {}), + "Column 'secret' is read, but the scan did not ask the catalog to " + "authorize it"); + TableLoadOptions read_context_load_26{path}; + ASSERT_OK_AND_ASSIGN(read_context, - ReadContextBuilder(path).SetReadFieldNames({"id", "salary"}).Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, - ReadAll(std::move(read_context), all_plan->Splits())); + ReadContextBuilder().SetReadFieldNames({"id", "salary"}).Finish()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr rows, + ReadAll(read_context_load_26, std::move(read_context), all_plan->Splits(), {})); AssertColumn(*rows, "id", "[1, 2, 3]"); - ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder(path).SetReadFieldNames({"id"}).Finish()); - ASSERT_NOK_WITH_MSG(ReadAll(std::move(read_context), masked_plan->Splits()), - "Query auth rules reference column 'phone' which the schema being read " - "exposes as 'name' (renamed since)"); - ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder(path).SetReadFieldNames({"id"}).Finish()); - ASSERT_NOK_WITH_MSG(ReadAll(std::move(read_context), folded_plan->Splits()), - "Query auth rules reference column 'phone' which the schema being read " - "exposes as 'name' (renamed since)"); + TableLoadOptions read_context_load_27{path}; + + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"id"}).Finish()); + ASSERT_NOK_WITH_MSG( + ReadAll(read_context_load_27, std::move(read_context), masked_plan->Splits(), {}), + "Query auth rules reference column 'phone' which the schema being read " + "exposes as 'name' (renamed since)"); + TableLoadOptions read_context_load_28{path}; + + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"id"}).Finish()); + ASSERT_NOK_WITH_MSG( + ReadAll(read_context_load_28, std::move(read_context), folded_plan->Splits(), {}), + "Query auth rules reference column 'phone' which the schema being read " + "exposes as 'name' (renamed since)"); } TEST(TableTest, TestQueryAuthTableAsksForTheRulesOfTheBranchPlanned) { @@ -905,14 +1097,15 @@ TEST(TableTest, TestQueryAuthTableAsksForTheRulesOfTheBranchPlanned) { CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); catalog->column_masking = {{"phone", R"({"name": "NULL"})"}}; - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path) - .WithCatalog(catalog, Identifier("foo", "bar")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ(Identifier("foo", "bar$branch_dev"), scan_context->GetIdentifier().value()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_load_29{path}; + std::map scan_context_options_29; + scan_context_load_29.catalog = catalog; + scan_context_load_29.identifier.emplace(Identifier("foo", "bar")); + scan_context_options_29[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(scan_context_load_29, std::move(scan_context), scan_context_options_29)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_FALSE(catalog->TableFileSystemRequests().empty()); for (const Identifier& requested : catalog->TableFileSystemRequests()) { @@ -923,10 +1116,13 @@ TEST(TableTest, TestQueryAuthTableAsksForTheRulesOfTheBranchPlanned) { ASSERT_EQ(std::vector({Identifier("foo", "bar$branch_dev")}), catalog->location_identifiers); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(path).AddOption(Options::BRANCH, "dev").Finish()); + TableLoadOptions read_context_load_30{path}; + std::map read_context_options_30; + read_context_options_30[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, - ReadAll(std::move(read_context), plan->Splits())); + ReadAll(read_context_load_30, std::move(read_context), plan->Splits(), + read_context_options_30)); AssertColumn(*rows, "id", "[7]"); AssertColumn(*rows, "phone", "[null]"); } @@ -949,23 +1145,29 @@ TEST(TableTest, TestQueryAuthTableIsNotSkippedThroughAStaleCachedSchema) { document.Accept(writer); const std::string stale_schema = buffer.GetString(); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path).SetTableSchema(stale_schema).Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions scan_context_load_31{path}; + + scan_context_load_31.table_schema_json = stale_schema; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_31, std::move(scan_context), {}), "without the catalog that serves its row filters and column masks"); ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); - ASSERT_OK_AND_ASSIGN( - scan_context, - ScanContextBuilder(path).WithCatalog(catalog, Identifier("foo", "bar")).Finish()); + TableLoadOptions scan_context_load_32{path}; + + scan_context_load_32.catalog = catalog; + scan_context_load_32.identifier.emplace(Identifier("foo", "bar")); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_32, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(path).SetTableSchema(stale_schema).Finish()); + TableLoadOptions read_context_load_33{path}; + + read_context_load_33.table_schema_json = stale_schema; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_load_33, std::move(read_context), {})); ASSERT_NOK_WITH_MSG( read->CreateReader( std::dynamic_pointer_cast(plan->Splits()[0])->GetSplit()), @@ -975,13 +1177,15 @@ TEST(TableTest, TestQueryAuthTableIsNotSkippedThroughAStaleCachedSchema) { .CreateTable(QueryAuthTableSchema(), /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::QUERY_AUTH_ENABLED, "false"}}) .status()); - ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder(path) - .WithBranch("dev") - .SetTableSchema(stale_schema) - .AddOption(Options::SCAN_FALLBACK_BRANCH, "main") - .Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), - "Cannot read fallback branch 'main', which enables 'query-auth.enabled'"); + TableLoadOptions read_context_load_34{path}; + std::map read_context_options_34; + read_context_options_34[Options::BRANCH] = "dev"; + read_context_load_34.table_schema_json = stale_schema; + read_context_options_34[Options::SCAN_FALLBACK_BRANCH] = "main"; + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_context_load_34, std::move(read_context), read_context_options_34), + "Cannot read fallback branch 'main', which enables 'query-auth.enabled'"); } TEST(TableTest, TestQueryAuthScanDoesNotPruneByTheRawValuesOfAMaskedColumn) { @@ -1000,23 +1204,26 @@ TEST(TableTest, TestQueryAuthScanDoesNotPruneByTheRawValuesOfAMaskedColumn) { auto read_with = [&](const std::shared_ptr& predicate, const std::shared_ptr& plan, bool enable_predicate_filter = true) -> Result> { + TableLoadOptions context_load_35{path}; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - ReadContextBuilder(path) + ReadContextBuilder() .SetPredicate(predicate) .EnablePredicateFilter(enable_predicate_filter) .Finish()); - return ReadAll(std::move(context), plan->Splits()); + return ReadAll(context_load_35, std::move(context), plan->Splits(), {}); }; const std::shared_ptr phone_is_139 = PredicateBuilder::Equal( /*field_index=*/2, "phone", FieldType::STRING, Literal(FieldType::STRING, "139", 3)); + TableLoadOptions scan_context_load_36{path}; + + scan_context_load_36.catalog = catalog; + scan_context_load_36.identifier.emplace(Identifier("foo", "bar")); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path) - .WithCatalog(catalog, Identifier("foo", "bar")) - .SetPredicate(phone_is_139) - .Finish()); + ScanContextBuilder().SetPredicate(phone_is_139).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_36, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_TRUE(plan->Splits().empty()); @@ -1068,38 +1275,48 @@ TEST(TableTest, TestQueryAuthReadRefusesAColumnTheScanDidNotAuthorize) { auto read_names = [&](bool enable_predicate_filter, const std::vector>& splits) -> Result> { + TableLoadOptions context_load_37{path}; + PAIMON_ASSIGN_OR_RAISE( std::unique_ptr context, - ReadContextBuilder(path) + ReadContextBuilder() .SetReadFieldNames({"name"}) .SetPredicate(PredicateBuilder::Equal(/*field_index=*/1, "phone", FieldType::STRING, Literal(FieldType::STRING, "139", 3))) .EnablePredicateFilter(enable_predicate_filter) .Finish()); - return ReadAll(std::move(context), splits); + return ReadAll(context_load_37, std::move(context), splits, {}); }; const std::string unauthorized = "Column 'phone' is read, but the scan did not ask the catalog to authorize it"; + TableLoadOptions name_scan_context_load_38{path}; + + name_scan_context_load_38.catalog = catalog; + name_scan_context_load_38.identifier.emplace(Identifier("foo", "bar")); ASSERT_OK_AND_ASSIGN(std::unique_ptr name_scan_context, - ScanContextBuilder(path) - .WithCatalog(catalog, Identifier("foo", "bar")) - .SetReadFieldNames({"name"}) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr name_scan, - TableScan::Create(std::move(name_scan_context))); + ScanContextBuilder().SetReadFieldNames({"name"}).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr name_scan, + TableScan::Create(name_scan_context_load_38, std::move(name_scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr name_plan, name_scan->CreatePlan()); ASSERT_EQ(std::optional>(std::vector{"name"}), catalog->auth_selects.back()); + TableLoadOptions all_columns_context_load_39{path}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr all_columns_context, - ReadContextBuilder(path).Finish()); + ReadContextBuilder().Finish()); ASSERT_NOK_WITH_MSG( - ReadAll(std::move(all_columns_context), name_plan->Splits()), + ReadAll(all_columns_context_load_39, std::move(all_columns_context), name_plan->Splits(), + {}), "Column 'id' is read, but the scan did not ask the catalog to authorize it"); + TableLoadOptions name_context_load_40{path}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr name_context, - ReadContextBuilder(path).SetReadFieldNames({"name"}).Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, - ReadAll(std::move(name_context), name_plan->Splits())); + ReadContextBuilder().SetReadFieldNames({"name"}).Finish()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr rows, + ReadAll(name_context_load_40, std::move(name_context), name_plan->Splits(), {})); AssertColumn(*rows, "name", R"(["alice", "bob", "carol"])"); ASSERT_NOK_WITH_MSG(read_names(/*enable_predicate_filter=*/true, name_plan->Splits()), unauthorized); @@ -1213,11 +1430,14 @@ TEST(TableTest, TestQueryAuthTableReadsNoRowsUnderAJavaAlwaysFalseFilter) { [&](const std::vector& read_fields) -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, PlanQueryAuth(path, catalog, /*predicate=*/nullptr, std::nullopt)); - ReadContextBuilder builder(path); + TableLoadOptions builder_load{path}; + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames(read_fields); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr read, + TableRead::Create(builder_load, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr chunks, @@ -1308,10 +1528,13 @@ TEST(TableTest, TestQueryAuthDoesNotBringBackRowsADeletionVectorDeletes) { dir->GetFileSystem(), path, "orc", data_messages[0], {{data_message->GetNewFilesIncrement().NewFiles()[0]->file_name, {1}}}, GetDefaultPool())); + TableLoadOptions commit_context_load_42{path}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - CommitContextBuilder(path, "commit_user").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + CommitContextBuilder("commit_user").Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_context_load_42, std::move(commit_context), {})); ASSERT_OK(commit->Commit({deletion_message}, /*commit_identifier=*/1)); ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, @@ -1335,11 +1558,14 @@ TEST(TableTest, TestQueryAuthAppliesTheRulesToTheColumnsADataEvolutionTableRebui {R"([[1, "alice", "13800001111", 100], [2, "bob", "13900002222", 200]])"})); const std::string path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + TableLoadOptions write_context_load_43{path}; + ASSERT_OK_AND_ASSIGN( std::unique_ptr write_context, - WriteContextBuilder(path, "commit_user").WithWriteSchema({"phone", "salary"}).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + WriteContextBuilder("commit_user").WithWriteSchema({"phone", "salary"}).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write, + FileStoreWrite::Create(write_context_load_43, std::move(write_context), {})); ASSERT_OK_AND_ASSIGN( std::unique_ptr batch, TestHelper::MakeRecordBatch(arrow::struct_({arrow::field("phone", arrow::utf8()), @@ -1359,10 +1585,13 @@ TEST(TableTest, TestQueryAuthAppliesTheRulesToTheColumnsADataEvolutionTableRebui file->AssignFirstRowId(0); } } + TableLoadOptions commit_context_load_44{path}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - CommitContextBuilder(path, "commit_user").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + CommitContextBuilder("commit_user").Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_context_load_44, std::move(commit_context), {})); ASSERT_OK(commit->Commit(messages, /*commit_identifier=*/1)); ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, @@ -1395,13 +1624,16 @@ TEST(TableTest, TestQueryAuthFiltersDictionaryEncodedParquetColumns) { bool enable_predicate_filter) -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, PlanQueryAuth(path, catalog, predicate, std::nullopt)); - ReadContextBuilder builder(path); + TableLoadOptions builder_load{path}; + std::map builder_options; + ReadContextBuilder builder; if (predicate != nullptr) { builder.SetPredicate(predicate).EnablePredicateFilter(enable_predicate_filter); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(builder_load, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch()); @@ -1482,10 +1714,12 @@ TEST(TableTest, TestQueryAuthScanReadsOnlyTheLocationOfTheAuthorizedTable) { CreateQueryAuthCatalog(*authorized, authorized_dir->GetFileSystem())); const Identifier identifier("foo", "bar"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr scan_context, - ScanContextBuilder(other->TablePath()).WithCatalog(catalog, identifier).Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions scan_context_load_46{other->TablePath()}; + + scan_context_load_46.catalog = catalog; + scan_context_load_46.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_46, std::move(scan_context), {}), fmt::format("Invalid: The scan targets {}, but the catalog keeps table " "foo.bar, whose query auth settings it would apply, at {}", other->TablePath(), authorized->TablePath())); @@ -1495,19 +1729,24 @@ TEST(TableTest, TestQueryAuthScanReadsOnlyTheLocationOfTheAuthorizedTable) { {authorized->TablePath() + "/", PathUtil::JoinPath(authorized_dir->Str() + "/", "foo.db//bar")}) { SCOPED_TRACE(path); - ASSERT_OK_AND_ASSIGN(scan_context, - ScanContextBuilder(path).WithCatalog(catalog, identifier).Finish()); + TableLoadOptions scan_context_load_47{path}; + + scan_context_load_47.catalog = catalog; + scan_context_load_47.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_47, std::move(scan_context), {})); ASSERT_OK(scan->CreatePlan().status()); } // The identifier is rebound to another table between the scan and its first plan. - ASSERT_OK_AND_ASSIGN( - scan_context, - ScanContextBuilder(authorized->TablePath()).WithCatalog(catalog, identifier).Finish()); + TableLoadOptions scan_context_load_48{authorized->TablePath()}; + + scan_context_load_48.catalog = catalog; + scan_context_load_48.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr rebound_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_48, std::move(scan_context), {})); catalog->table_location = other->TablePath(); const size_t auth_requests = catalog->auth_identifiers.size(); ASSERT_NOK_WITH_MSG(rebound_scan->CreatePlan(), @@ -1517,11 +1756,13 @@ TEST(TableTest, TestQueryAuthScanReadsOnlyTheLocationOfTheAuthorizedTable) { ASSERT_EQ(auth_requests + 1, catalog->auth_identifiers.size()); catalog->table_location.clear(); - ASSERT_OK_AND_ASSIGN( - scan_context, - ScanContextBuilder(authorized->TablePath()).WithCatalog(catalog, identifier).Finish()); + TableLoadOptions scan_context_load_49{authorized->TablePath()}; + + scan_context_load_49.catalog = catalog; + scan_context_load_49.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); ASSERT_NOK_WITH_MSG( - TableScan::Create(std::move(scan_context)), + TableScan::Create(scan_context_load_49, std::move(scan_context), {}), "Cannot check that " + authorized->TablePath() + " is the location of table foo.bar"); } @@ -1555,13 +1796,16 @@ TEST(TableTest, TestQueryAuthReadRejectsSplitsAuthorizedForAnotherTable) { splits.push_back(std::move(restored)); } for (bool with_catalog : {false, true}) { - ReadContextBuilder builder(other->TablePath()); + TableLoadOptions builder_load{other->TablePath()}; + std::map builder_options; + ReadContextBuilder builder; if (with_catalog) { - builder.WithCatalog(other_catalog, Identifier("foo", "bar")); + builder_load.catalog = other_catalog; + builder_load.identifier.emplace(Identifier("foo", "bar")); } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + TableRead::Create(builder_load, std::move(context), builder_options)); ASSERT_FALSE(splits.empty()); ASSERT_NOK_WITH_MSG(read->CreateReader(splits.front()), "split was authorized for"); ASSERT_NOK_WITH_MSG(read->CreateReader(splits), "split was authorized for"); @@ -1570,10 +1814,11 @@ TEST(TableTest, TestQueryAuthReadRejectsSplitsAuthorizedForAnotherTable) { // A worker can use an equivalent spelling of the authorized table's location. for (const std::string& path : {authorized->TablePath() + "/", "file://" + authorized->TablePath()}) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(path).Finish()); + TableLoadOptions context_load_51{path}; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, - ReadAll(std::move(context), splits)); + ReadAll(context_load_51, std::move(context), splits, {})); AssertColumn(*rows, "phone", R"(["13800001111", "13900002222", null])"); } } @@ -1592,9 +1837,13 @@ TEST(TableTest, TestQueryAuthReadRejectsSplitsAuthorizedForAnotherBranch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanQueryAuth(path, catalog, /*predicate=*/nullptr, std::nullopt)); ASSERT_FALSE(plan->Splits().empty()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(path).WithBranch("dev").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, TableRead::Create(std::move(context))); + TableLoadOptions context_load_52{path}; + std::map context_options_52; + context_options_52[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(context_load_52, std::move(context), context_options_52)); ASSERT_NOK_WITH_MSG(read->CreateReader(plan->Splits().front()), "split was authorized for table " + path + " (branch main)"); ASSERT_NOK_WITH_MSG(read->CreateReader(plan->Splits()), @@ -1649,25 +1898,39 @@ TEST(TableTest, TestCatalogDisablesQueryAuthOnlyForTheTableAtItsLocation) { {std::shared_ptr(auth_catalog), std::shared_ptr(plain_catalog)}) { for (const std::string& suffix : {std::string(), std::string("$files")}) { SCOPED_TRACE(suffix); + TableLoadOptions scan_context_load_53{other->TablePath() + suffix}; + + scan_context_load_53.catalog = catalog; + scan_context_load_53.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(other->TablePath() + suffix) - .WithCatalog(catalog, identifier) - .Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), mismatch("scan")); + ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableScan::Create(scan_context_load_53, std::move(scan_context), {}), + mismatch("scan")); + TableLoadOptions read_context_load_54{other->TablePath() + suffix}; + + read_context_load_54.catalog = catalog; + read_context_load_54.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(other->TablePath() + suffix) - .WithCatalog(catalog, identifier) - .Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), mismatch("read")); - - ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder(disabled->TablePath() + suffix) - .WithCatalog(catalog, identifier) - .Finish()); - ASSERT_OK(TableScan::Create(std::move(scan_context)).status()); - ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder(disabled->TablePath() + suffix) - .WithCatalog(catalog, identifier) - .Finish()); - ASSERT_OK(TableRead::Create(std::move(read_context)).status()); + ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_context_load_54, std::move(read_context), {}), + mismatch("read")); + + TableLoadOptions scan_context_load_55{disabled->TablePath() + suffix}; + + scan_context_load_55.catalog = catalog; + scan_context_load_55.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); + ASSERT_OK( + TableScan::Create(scan_context_load_55, std::move(scan_context), {}).status()); + TableLoadOptions read_context_load_56{disabled->TablePath() + suffix}; + + read_context_load_56.catalog = catalog; + read_context_load_56.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().Finish()); + ASSERT_OK( + TableRead::Create(read_context_load_56, std::move(read_context), {}).status()); } } ASSERT_TRUE(auth_catalog->auth_identifiers.empty()); @@ -1685,110 +1948,161 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { const std::string disabled = "Invalid: Option 'query-auth.enabled' is not allowed to be disabled dynamically"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path).Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions scan_context_load_57{path}; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_57, std::move(scan_context), {}), "NotImplemented: Cannot read a table with 'query-auth.enabled=true' " "without the catalog"); auto plain_catalog = std::make_shared(); plain_catalog->SetFileSystem(dir->GetFileSystem()); plain_catalog->SetTableSchema(checked_pointer_cast(table_schema.value())); - ASSERT_OK_AND_ASSIGN(scan_context, - ScanContextBuilder(path).WithCatalog(plain_catalog, identifier).Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions scan_context_load_58{path}; + + scan_context_load_58.catalog = plain_catalog; + scan_context_load_58.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_58, std::move(scan_context), {}), "its catalog does not support the query auth interface"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr disabling_scan_context, - ScanContextBuilder(path).AddOption(Options::QUERY_AUTH_ENABLED, "false").Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(disabling_scan_context)), disabled); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr disabling_read_context, - ReadContextBuilder(path).AddOption(Options::QUERY_AUTH_ENABLED, "false").Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(disabling_read_context)), disabled); + TableLoadOptions disabling_scan_context_load_59{path}; + std::map disabling_scan_context_options_59; + disabling_scan_context_options_59[Options::QUERY_AUTH_ENABLED] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr disabling_scan_context, + ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableScan::Create(disabling_scan_context_load_59, std::move(disabling_scan_context), + disabling_scan_context_options_59), + disabled); + TableLoadOptions disabling_read_context_load_60{path}; + std::map disabling_read_context_options_60; + disabling_read_context_options_60[Options::QUERY_AUTH_ENABLED] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr disabling_read_context, + ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableRead::Create(disabling_read_context_load_60, std::move(disabling_read_context), + disabling_read_context_options_60), + disabled); auto explicit_read_schema = std::make_unique<::ArrowSchema>(); ASSERT_TRUE(arrow::ExportSchema(*arrow::schema({arrow::field("name", arrow::utf8())}), explicit_read_schema.get()) .ok()); + TableLoadOptions explicit_schema_read_context_load_61{path}; + ASSERT_OK_AND_ASSIGN( std::unique_ptr explicit_schema_read_context, - ReadContextBuilder(path).SetReadSchema(std::move(explicit_read_schema)).Finish()); - Status explicit_schema_status = - TableRead::Create(std::move(explicit_schema_read_context)).status(); + ReadContextBuilder().SetReadSchema(std::move(explicit_read_schema)).Finish()); + Status explicit_schema_status = TableRead::Create(explicit_schema_read_context_load_61, + std::move(explicit_schema_read_context), {}) + .status(); ASSERT_TRUE(explicit_schema_status.IsNotImplemented()) << explicit_schema_status.ToString(); ASSERT_NOK_WITH_MSG(explicit_schema_status, "An explicit read schema is not supported on a query-auth table"); + TableLoadOptions search_read_context_load_62{path}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr search_read_context, - ReadContextBuilder(path).EnablePreciseFileIndexEvaluate(true).Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(search_read_context)), - "Search is not supported on a query-auth table"); + ReadContextBuilder().EnablePreciseFileIndexEvaluate(true).Finish()); + ASSERT_NOK_WITH_MSG( + TableRead::Create(search_read_context_load_62, std::move(search_read_context), {}), + "Search is not supported on a query-auth table"); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + TableLoadOptions realtime_read_context_load_63{path}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_read_context, - ReadContextBuilder(path).WithRealtimeContext(realtime_context).Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(realtime_read_context)), - "Real-time reads are not supported on a query-auth table"); + ReadContextBuilder().WithRealtimeContext(realtime_context).Finish()); + ASSERT_NOK_WITH_MSG( + TableRead::Create(realtime_read_context_load_63, std::move(realtime_read_context), {}), + "Real-time reads are not supported on a query-auth table"); const std::string row_id = SpecialFields::RowId().Name(); const std::string not_in_schema = fmt::format("NotImplemented: Column '{}' is not a column of the table schema", row_id); + TableLoadOptions row_id_read_context_load_64{path}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr row_id_read_context, - ReadContextBuilder(path).SetReadFieldNames({"id", row_id}).Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(row_id_read_context)), not_in_schema); + ReadContextBuilder().SetReadFieldNames({"id", row_id}).Finish()); + ASSERT_NOK_WITH_MSG( + TableRead::Create(row_id_read_context_load_64, std::move(row_id_read_context), {}), + not_in_schema); + TableLoadOptions row_id_by_id_read_context_load_65{path}; + ASSERT_OK_AND_ASSIGN( std::unique_ptr row_id_by_id_read_context, - ReadContextBuilder(path).SetReadFieldIds({0, SpecialFields::RowId().Id()}).Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(row_id_by_id_read_context)), + ReadContextBuilder().SetReadFieldIds({0, SpecialFields::RowId().Id()}).Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(row_id_by_id_read_context_load_65, + std::move(row_id_by_id_read_context), {}), fmt::format("NotImplemented: Field {} is not a column of the table schema", SpecialFields::RowId().Id())); + TableLoadOptions row_id_filter_read_context_load_66{path}; + ASSERT_OK_AND_ASSIGN( std::unique_ptr row_id_filter_read_context, - ReadContextBuilder(path) + ReadContextBuilder() .SetReadFieldNames({"id"}) .SetPredicate(PredicateBuilder::IsNotNull(/*field_index=*/1, row_id, FieldType::BIGINT)) .EnablePredicateFilter(true) .Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(row_id_filter_read_context)), not_in_schema); + ASSERT_NOK_WITH_MSG(TableRead::Create(row_id_filter_read_context_load_66, + std::move(row_id_filter_read_context), {}), + not_in_schema); + TableLoadOptions row_id_unfiltered_read_context_load_67{path}; + ASSERT_OK_AND_ASSIGN( std::unique_ptr row_id_unfiltered_read_context, - ReadContextBuilder(path) + ReadContextBuilder() .SetReadFieldNames({"id"}) .SetPredicate(PredicateBuilder::IsNotNull(/*field_index=*/1, row_id, FieldType::BIGINT)) .EnablePredicateFilter(false) .Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(row_id_unfiltered_read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(row_id_unfiltered_read_context_load_67, + std::move(row_id_unfiltered_read_context), {}), not_in_schema); ASSERT_OK_AND_ASSIGN(std::shared_ptr auth_catalog, CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); - ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder(path) - .WithCatalog(auth_catalog, identifier) - .SetReadFieldNames({"id", row_id}) - .Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), not_in_schema); - ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder(path) - .WithCatalog(auth_catalog, identifier) + TableLoadOptions scan_context_load_68{path}; + + scan_context_load_68.catalog = auth_catalog; + scan_context_load_68.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, + ScanContextBuilder().SetReadFieldNames({"id", row_id}).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_68, std::move(scan_context), {}), + not_in_schema); + TableLoadOptions scan_context_load_69{path}; + + scan_context_load_69.catalog = auth_catalog; + scan_context_load_69.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder() .SetPredicate(PredicateBuilder::IsNotNull( /*field_index=*/4, row_id, FieldType::BIGINT)) .Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), not_in_schema); - ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder(path) - .WithCatalog(auth_catalog, identifier) - .SetPartitionFilter({{{"gone", "1"}}}) - .Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_69, std::move(scan_context), {}), + not_in_schema); + TableLoadOptions scan_context_load_70{path}; + + scan_context_load_70.catalog = auth_catalog; + scan_context_load_70.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, + ScanContextBuilder().SetPartitionFilter({{{"gone", "1"}}}).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_70, std::move(scan_context), {}), "NotImplemented: Column 'gone' is not a column of the table schema"); + TableLoadOptions scan_context_load_71{path}; + + scan_context_load_71.catalog = auth_catalog; + scan_context_load_71.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN( - scan_context, ScanContextBuilder(path) - .WithCatalog(auth_catalog, identifier) + scan_context, ScanContextBuilder() .SetGlobalIndexResult(BitmapGlobalIndexResult::FromRanges({Range(0, 1)})) .Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_71, std::move(scan_context), {}), "Search is not supported on a query-auth table"); - ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder(path) - .WithCatalog(auth_catalog, identifier) - .WithRealtimeContext(realtime_context) - .Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions scan_context_load_72{path}; + + scan_context_load_72.catalog = auth_catalog; + scan_context_load_72.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, + ScanContextBuilder().WithRealtimeContext(realtime_context).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_72, std::move(scan_context), {}), "Real-time reads are not supported on a query-auth table"); ASSERT_TRUE(auth_catalog->auth_identifiers.empty()); const std::string global_index_refused = @@ -1827,11 +2141,13 @@ TEST(TableTest, TestQueryAuthRefusesFallbackReadsAcrossBranches) { .CreateTable(QueryAuthTableSchema(), /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::QUERY_AUTH_ENABLED, enabled}}) .status()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr context, - ReadContextBuilder(path).AddOption(Options::SCAN_FALLBACK_BRANCH, branch).Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), - "Fallback reads are not supported on a query-auth table"); + TableLoadOptions context_load_73{path}; + std::map context_options_73; + context_options_73[Options::SCAN_FALLBACK_BRANCH] = branch; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableRead::Create(context_load_73, std::move(context), context_options_73), + "Fallback reads are not supported on a query-auth table"); } const std::string fallback_path = PathUtil::JoinPath(dir->Str(), "fallback_tbl"); @@ -1844,13 +2160,16 @@ TEST(TableTest, TestQueryAuthRefusesFallbackReadsAcrossBranches) { .CreateTable(fallback_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{{Options::QUERY_AUTH_ENABLED, "true"}}) .status()); + TableLoadOptions fallback_read_context_load_74{fallback_path}; + std::map fallback_read_context_options_74; + fallback_read_context_options_74[Options::SCAN_FALLBACK_BRANCH] = "fallback"; ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_read_context, - ReadContextBuilder(fallback_path) - .AddOption(Options::SCAN_FALLBACK_BRANCH, "fallback") - .Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(fallback_read_context)), - "Cannot read fallback branch 'fallback', which enables " - "'query-auth.enabled', of a table that does not"); + ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableRead::Create(fallback_read_context_load_74, std::move(fallback_read_context), + fallback_read_context_options_74), + "Cannot read fallback branch 'fallback', which enables " + "'query-auth.enabled', of a table that does not"); } TEST(TableTest, TestQueryAuthSystemTablesValidateTheBranchRead) { @@ -1894,37 +2213,62 @@ TEST(TableTest, TestQueryAuthSystemTablesValidateTheBranchRead) { ASSERT_NOK_WITH_MSG( SystemTableLoader::LoadFromPath(dir->GetFileSystem(), path + "$branch_dev$" + name, {}), refused); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr scan_context, - ScanContextBuilder(system_path).AddOption(Options::BRANCH, "dev").Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), refused); + TableLoadOptions scan_context_load_75{system_path}; + std::map scan_context_options_75; + scan_context_options_75[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_75, std::move(scan_context), + scan_context_options_75), + refused); + TableLoadOptions read_context_load_76{system_path}; + std::map read_context_options_76; + read_context_options_76[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(system_path).WithBranch("dev").Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), refused); + ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_load_76, std::move(read_context), + read_context_options_76), + refused); const std::string restricted_path = path + "$branch_restricted$" + name; - ASSERT_OK_AND_ASSIGN( - scan_context, - ScanContextBuilder(restricted_path).WithCatalog(catalog, identifier).Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), refused); - ASSERT_OK_AND_ASSIGN( - read_context, - ReadContextBuilder(restricted_path).WithCatalog(catalog, identifier).Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), refused); + TableLoadOptions scan_context_load_77{restricted_path}; + + scan_context_load_77.catalog = catalog; + scan_context_load_77.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_77, std::move(scan_context), {}), + refused); + TableLoadOptions read_context_load_78{restricted_path}; + + read_context_load_78.catalog = catalog; + read_context_load_78.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_load_78, std::move(read_context), {}), + refused); ASSERT_EQ(Identifier("foo", "bar$branch_restricted"), catalog->LoadTableSchemaIdentifiers().back()); } + TableLoadOptions main_read_context_load_79{path + "$files"}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_read_context, - ReadContextBuilder(path + "$files").Finish()); - ASSERT_OK(TableRead::Create(std::move(main_read_context)).status()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr main_scan_context, - ScanContextBuilder(path + "$files").WithCatalog(catalog, identifier).Finish()); - ASSERT_OK(TableScan::Create(std::move(main_scan_context)).status()); - ASSERT_OK_AND_ASSIGN( - main_read_context, - ReadContextBuilder(path + "$files").WithCatalog(catalog, identifier).Finish()); - ASSERT_OK(TableRead::Create(std::move(main_read_context)).status()); + ReadContextBuilder().Finish()); + ASSERT_OK( + TableRead::Create(main_read_context_load_79, std::move(main_read_context), {}).status()); + TableLoadOptions main_scan_context_load_80{path + "$files"}; + + main_scan_context_load_80.catalog = catalog; + main_scan_context_load_80.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan_context, + ScanContextBuilder().Finish()); + ASSERT_OK( + TableScan::Create(main_scan_context_load_80, std::move(main_scan_context), {}).status()); + TableLoadOptions main_read_context_load_81{path + "$files"}; + + main_read_context_load_81.catalog = catalog; + main_read_context_load_81.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(main_read_context, ReadContextBuilder().Finish()); + ASSERT_OK( + TableRead::Create(main_read_context_load_81, std::move(main_read_context), {}).status()); ASSERT_NOK_WITH_MSG( SystemTableLoader::LoadFromPath(dir->GetFileSystem(), path + "$branch_dev$files", {{Options::BRANCH, "other"}}), @@ -1943,11 +2287,13 @@ TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { // A catalog write reads the current schema from the catalog on the main branch. catalog->SetTableSchema(table->GetSchema()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr ctx, - WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + std::map ctx_builder_18974_options; + WriteContextBuilder ctx_builder_18974("commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder_18974.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_5, + table->Copy(ctx_builder_18974_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, - table->NewWrite(std::move(ctx))); + configured_table_5->NewWrite(std::move(ctx))); } TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { @@ -1962,11 +2308,13 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { // Like a write, a catalog commit reads the current schema from the catalog. catalog->SetTableSchema(table->GetSchema()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr ctx, - CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + std::map ctx_builder_19886_options; + CommitContextBuilder ctx_builder_19886("commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder_19886.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_6, + table->Copy(ctx_builder_19886_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, - table->NewCommit(std::move(ctx))); + configured_table_6->NewCommit(std::move(ctx))); } TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { @@ -1999,27 +2347,46 @@ TEST(TableTest, TestCreateBuildsTheOperator) { // `TableScan::Create()` and friends build the operator straight from the context, which // carries the catalog and identifier it resolves the table through. - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, - ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + TableLoadOptions scan_ctx_builder_load_options{dir->Str()}; + std::map scan_ctx_builder_options; + ScanContextBuilder scan_ctx_builder; + scan_ctx_builder_load_options.catalog = catalog; + scan_ctx_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - TableScan::Create(std::move(scan_ctx))); - - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, - ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + TableScan::Create(scan_ctx_builder_load_options, std::move(scan_ctx), + scan_ctx_builder_options)); + + TableLoadOptions read_ctx_builder_load_options{dir->Str()}; + std::map read_ctx_builder_options; + ReadContextBuilder read_ctx_builder; + read_ctx_builder_load_options.catalog = catalog; + read_ctx_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - TableRead::Create(std::move(read_ctx))); - - ASSERT_OK_AND_ASSIGN( - std::unique_ptr write_ctx, - WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + TableRead::Create(read_ctx_builder_load_options, std::move(read_ctx), + read_ctx_builder_options)); + + TableLoadOptions write_ctx_builder_load_options{dir->Str()}; + std::map write_ctx_builder_options; + WriteContextBuilder write_ctx_builder("commit-user"); + write_ctx_builder_load_options.catalog = catalog; + write_ctx_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_ctx, write_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, - FileStoreWrite::Create(std::move(write_ctx))); - + FileStoreWrite::Create(write_ctx_builder_load_options, + std::move(write_ctx), write_ctx_builder_options)); + + TableLoadOptions commit_ctx_builder_load_options{dir->Str()}; + std::map commit_ctx_builder_options; + CommitContextBuilder commit_ctx_builder("commit-user"); + commit_ctx_builder_load_options.catalog = catalog; + commit_ctx_builder_load_options.identifier.emplace(identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_ctx, commit_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_ctx, - CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_ctx))); + [[maybe_unused]] std::unique_ptr commit, + FileStoreCommit::Create(commit_ctx_builder_load_options, std::move(commit_ctx), + commit_ctx_builder_options)); } TEST(TableTest, TestLoadedSchemaAndPathAreUsedWithoutReloading) { @@ -2033,14 +2400,21 @@ TEST(TableTest, TestLoadedSchemaAndPathAreUsedWithoutReloading) { ASSERT_EQ(schema, table->GetSchema()); ASSERT_EQ((std::vector{"id"}), schema->FieldNames()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr read_context, - ReadContextBuilder(dir->Str() + "/other").SetReadFieldNames({"id"}).Finish()); + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder.SetReadFieldNames({"id"}).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_7, + table->Copy(read_context_builder_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - table->NewRead(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(dir->Str() + "/other").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + configured_table_7->NewRead(std::move(read_context))); + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_8, + table->Copy(scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + configured_table_8->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_TRUE(plan->Splits().empty()); } @@ -2053,21 +2427,51 @@ TEST(TableTest, TestContextOptionsCannotChangeLoadedTypeOrBranch) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); for (const auto& options : {std::map{{Options::TYPE, "format-table"}}, std::map{{Options::BRANCH, "dev"}}}) { + std::map scan_context_builder_25016_options; + ScanContextBuilder scan_context_builder_25016; + scan_context_builder_25016_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(dir->Str()).SetOptions(options).Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), + scan_context_builder_25016.Finish()); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->Copy(scan_context_builder_25016_options)); + return configured->NewScan(std::move(scan_context)); + })(), "contradicts the loaded table"); + std::map read_context_builder_25926_options; + ReadContextBuilder read_context_builder_25926; + read_context_builder_25926_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(dir->Str()).SetOptions(options).Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), + read_context_builder_25926.Finish()); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->Copy(read_context_builder_25926_options)); + return configured->NewRead(std::move(read_context)); + })(), "contradicts the loaded table"); + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("user"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - WriteContextBuilder(dir->Str(), "user").SetOptions(options).Finish()); - ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), + write_context_builder.Finish()); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(write_context_builder_options)); + return configured->NewWrite(std::move(write_context)); + })(), "contradicts the loaded table"); + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("user"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - CommitContextBuilder(dir->Str(), "user").SetOptions(options).Finish()); - ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), + commit_context_builder.Finish()); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(commit_context_builder_options)); + return configured->NewCommit(std::move(commit_context)); + })(), "contradicts the loaded table"); } } @@ -2081,15 +2485,25 @@ TEST(TableTest, TestOperationOptionsDoNotMutateTheLoadedTable) { auto schema = std::dynamic_pointer_cast(table->GetSchema()); ASSERT_NE(nullptr, schema); auto options = schema->Options(); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr invalid_context, - ScanContextBuilder(dir->Str()).SetOptions({{Options::BUCKET, "0"}}).Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(invalid_context)), "bucket=0"); + std::map invalid_context_builder_options; + ScanContextBuilder invalid_context_builder; + invalid_context_builder_options = {{Options::BUCKET, "0"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_context, + invalid_context_builder.Finish()); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(invalid_context_builder_options)); + return configured->NewScan(std::move(invalid_context)); + })(), + "bucket=0"); ASSERT_EQ(options, schema->Options()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ScanContextBuilder(dir->Str()).Finish()); + std::map context_builder_27781_options; + ScanContextBuilder context_builder_27781; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_27781.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_14, + table->Copy(context_builder_27781_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - table->NewScan(std::move(context))); + configured_table_14->NewScan(std::move(context))); } TEST(TableTest, TestPathIdentifierPreservesBranch) { @@ -2120,16 +2534,28 @@ TEST(TableTest, TestGlobalSystemTableRetainsCatalog) { ASSERT_EQ("sys.tables", table->FullName()); ASSERT_EQ("sys.tables", table->Uuid()); ASSERT_EQ(nullptr, std::dynamic_pointer_cast(table->GetSchema())); + std::map scan_context_builder_28121_options; + ScanContextBuilder scan_context_builder_28121; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(table->GetRootPath()).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + scan_context_builder_28121.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_15, + table->Copy(scan_context_builder_28121_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + configured_table_15->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + std::map read_context_builder_29694_options; + ReadContextBuilder read_context_builder_29694; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(table->GetRootPath()).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + read_context_builder_29694.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_16, + table->Copy(read_context_builder_29694_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + configured_table_16->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); catalog.reset(); table.reset(); + configured_table_15.reset(); + configured_table_16.reset(); ASSERT_FALSE(weak_catalog.expired()); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); ASSERT_TRUE(BatchReader::IsEofBatch(batch)); @@ -2154,19 +2580,31 @@ TEST(TableTest, TestMetadataSystemTablesRejectConflictingBranches) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db", name))); for (const std::string requested : {"main", "dev"}) { + std::map read_context_options_82; + read_context_options_82[Options::BRANCH] = requested; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(path).WithBranch(requested).Finish()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr scan_context, - ScanContextBuilder(path).AddOption(Options::BRANCH, requested).Finish()); - auto read = table->NewRead(std::move(read_context)); - auto scan = table->NewScan(std::move(scan_context)); + ReadContextBuilder().Finish()); + + std::map scan_context_options_83; + scan_context_options_83[Options::BRANCH] = requested; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder().Finish()); + auto read = ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(read_context_options_82)); + return configured->NewRead(std::move(read_context)); + })(); + auto scan = ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(scan_context_options_83)); + return configured->NewScan(std::move(scan_context)); + })(); if (requested == branch) { ASSERT_OK(read); ASSERT_OK(scan); } else { - ASSERT_NOK_WITH_MSG(read, "context branch contradicts the loaded table"); - ASSERT_NOK_WITH_MSG(scan, "context branch contradicts the loaded table"); + ASSERT_NOK_WITH_MSG(read, "copy branch contradicts the loaded table"); + ASSERT_NOK_WITH_MSG(scan, "copy branch contradicts the loaded table"); } } } @@ -2179,20 +2617,18 @@ TEST(TableTest, TestOperatorFactoryPreservesLoadedSchemaCache) { ASSERT_TRUE(dir); ASSERT_OK(CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); std::shared_ptr
table; - if (commit) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - CommitContextBuilder(dir->Str(), "user").Finish()); - ASSERT_OK_AND_ASSIGN(table, TableFactory::Create(context.get())); - } else { - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - WriteContextBuilder(dir->Str(), "user").Finish()); - ASSERT_OK_AND_ASSIGN(table, TableFactory::Create(context.get())); - } + ASSERT_OK_AND_ASSIGN(table, TableFactory::Create(TableLoadOptions{dir->Str()}, {}, + commit ? "commit" : "write")); auto managed = checked_pointer_cast(table); ASSERT_OK(dir->GetFileSystem()->Delete(dir->Str() + "/schema/schema-0", false)); // The factory already read schema-0. Repeated accesses must use its warmed cache. ASSERT_OK(managed->GetSchemaManager()->ReadSchema(0)); ASSERT_OK(managed->GetSchemaManager()->ReadSchema(0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + table->Copy({{Options::TARGET_FILE_ROW_NUM, "1"}})); + auto copied = checked_pointer_cast(copy); + ASSERT_EQ(managed->GetSchemaManager(), copied->GetSchemaManager()); + ASSERT_OK(copied->GetSchemaManager()->ReadSchema(0)); } } @@ -2210,49 +2646,78 @@ TEST(TableTest, TestMainBranchAliasesUseMainSchemaForAllOperatorEntryPoints) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(identifier)); for (bool static_factory : {false, true}) { SCOPED_TRACE(alias + (static_factory ? ":static" : ":loaded")); - ScanContextBuilder builder(path); - builder.WithCatalog(catalog, identifier); + TableLoadOptions builder_load{path}; + std::map builder_options; + ScanContextBuilder builder; + builder_load.catalog = catalog; + builder_load.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); std::unique_ptr scan; if (static_factory) { - ASSERT_OK_AND_ASSIGN(scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + scan, TableScan::Create(builder_load, std::move(context), builder_options)); } else { - ASSERT_OK_AND_ASSIGN(scan, table->NewScan(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_19, + table->Copy(builder_options)); + ASSERT_OK_AND_ASSIGN(scan, configured_table_19->NewScan(std::move(context))); } ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_TRUE(plan->Splits().empty()); - ReadContextBuilder read_builder(path); - read_builder.WithCatalog(catalog, identifier); + TableLoadOptions read_builder_load{path}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_load.catalog = catalog; + read_builder_load.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); std::unique_ptr read; if (static_factory) { - ASSERT_OK_AND_ASSIGN(read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(read, + TableRead::Create(read_builder_load, std::move(read_context), + read_builder_options)); } else { - ASSERT_OK_AND_ASSIGN(read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_20, + table->Copy(read_builder_options)); + ASSERT_OK_AND_ASSIGN(read, configured_table_20->NewRead(std::move(read_context))); } - WriteContextBuilder write_builder(path, "user"); - write_builder.WithCatalog(catalog, identifier); + TableLoadOptions write_builder_load{path}; + std::map write_builder_options; + WriteContextBuilder write_builder("user"); + write_builder_load.catalog = catalog; + write_builder_load.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); std::unique_ptr write; if (static_factory) { - ASSERT_OK_AND_ASSIGN(write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + write, FileStoreWrite::Create(write_builder_load, std::move(write_context), + write_builder_options)); } else { - ASSERT_OK_AND_ASSIGN(write, table->NewWrite(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_21, + table->Copy(write_builder_options)); + ASSERT_OK_AND_ASSIGN(write, + configured_table_21->NewWrite(std::move(write_context))); } ASSERT_OK(write->Close()); - CommitContextBuilder commit_builder(path, "user"); - commit_builder.WithCatalog(catalog, identifier); + TableLoadOptions commit_builder_load{path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("user"); + commit_builder_load.catalog = catalog; + commit_builder_load.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); std::unique_ptr commit; if (static_factory) { - ASSERT_OK_AND_ASSIGN(commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + commit, FileStoreCommit::Create(commit_builder_load, std::move(commit_context), + commit_builder_options)); } else { - ASSERT_OK_AND_ASSIGN(commit, table->NewCommit(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_22, + table->Copy(commit_builder_options)); + ASSERT_OK_AND_ASSIGN(commit, + configured_table_22->NewCommit(std::move(commit_context))); } } } @@ -2270,20 +2735,31 @@ TEST(TableTest, TestExplicitContextBranchMustMatchLoadedTable) { {std::optional(), std::optional("main"), std::optional("dev"), std::optional(" ")}) { SCOPED_TRACE(table_branch + ":" + requested.value_or("unset")); - WriteContextBuilder write_builder(table->GetRootPath(), "user"); - ReadContextBuilder read_builder(table->GetRootPath()); + std::map write_builder_options; + WriteContextBuilder write_builder("user"); + std::map read_builder_options; + ReadContextBuilder read_builder; if (requested) { - write_builder.WithBranch(*requested); - read_builder.WithBranch(*requested); + write_builder_options["branch"] = *requested; + + read_builder_options["branch"] = *requested; } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - auto write = table->NewWrite(std::move(write_context)); - auto read = table->NewRead(std::move(read_context)); + auto write = ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(write_builder_options)); + return configured->NewWrite(std::move(write_context)); + })(); + auto read = ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(read_builder_options)); + return configured->NewRead(std::move(read_context)); + })(); if (requested && BranchManager::NormalizeBranch(*requested) != table_branch) { - ASSERT_NOK_WITH_MSG(write, "context branch contradicts the loaded table"); - ASSERT_NOK_WITH_MSG(read, "context branch contradicts the loaded table"); + ASSERT_NOK_WITH_MSG(write, "copy branch contradicts the loaded table"); + ASSERT_NOK_WITH_MSG(read, "copy branch contradicts the loaded table"); } else { ASSERT_OK(write); ASSERT_OK(read); @@ -2323,27 +2799,42 @@ TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { std::map{{Options::BRANCH, "main"}}}) { const std::string branch = overrides.empty() ? "rt" : overrides.at(Options::BRANCH); SCOPED_TRACE(branch); + TableLoadOptions scan_context_builder_30072_load_options{path}; + std::map scan_context_builder_30072_options; + ScanContextBuilder scan_context_builder_30072; + scan_context_builder_30072_load_options.table_schema_json = schema_json; + scan_context_builder_30072_options = overrides; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + scan_context_builder_30072.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr scan_context, - ScanContextBuilder(path).SetTableSchema(schema_json).SetOptions(overrides).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + std::unique_ptr scan, + TableScan::Create(scan_context_builder_30072_load_options, std::move(scan_context), + scan_context_builder_30072_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(branch == "rt" ? 1 : 2, plan->SnapshotId()); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr read_context, - ReadContextBuilder(path).SetTableSchema(schema_json).SetOptions(overrides).Finish()); + TableLoadOptions read_context_builder_34363_load_options{path}; + std::map read_context_builder_34363_options; + ReadContextBuilder read_context_builder_34363; + read_context_builder_34363_load_options.table_schema_json = schema_json; + read_context_builder_34363_options = overrides; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder_34363.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, - TableFactory::Create(read_context.get())); + TableFactory::Create(read_context_builder_34363_load_options, + read_context_builder_34363_options, "read")); ASSERT_EQ(branch, std::dynamic_pointer_cast(loaded)->CurrentBranch()); + ASSERT_EQ(BranchManager::BranchPath(path, branch), + checked_pointer_cast(loaded)->GetSchemaManager()->BranchPath()); // Explicit non-main loads keep using that branch's persisted schema. const auto& expected_options = overrides.count(Options::BRANCH) != 0 && branch == "rt" ? schema.value()->Options() : preloaded->Options(); ASSERT_EQ(expected_options, std::dynamic_pointer_cast(loaded->GetSchema())->Options()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_25, + loaded->Copy(read_context_builder_34363_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - loaded->NewRead(std::move(read_context))); + configured_table_25->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -2353,10 +2844,16 @@ TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { ASSERT_TRUE(snapshot); ASSERT_EQ(snapshot->TotalRecordCount(), rows->length()); } - ASSERT_OK_AND_ASSIGN( - std::unique_ptr explicit_main, - ReadContextBuilder(path).SetTableSchema(schema_json).WithBranch("main").Finish()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, TableFactory::Create(explicit_main.get())); + TableLoadOptions explicit_main_builder_load_options{path}; + std::map explicit_main_builder_options; + ReadContextBuilder explicit_main_builder; + explicit_main_builder_load_options.table_schema_json = schema_json; + explicit_main_builder_options["branch"] = "main"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr explicit_main, + explicit_main_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + TableFactory::Create(explicit_main_builder_load_options, + explicit_main_builder_options, "read")); ASSERT_EQ("main", std::dynamic_pointer_cast(loaded)->CurrentBranch()); } @@ -2378,23 +2875,31 @@ TEST(TableTest, TestResolvedStoreUsesOptionsBranch) { ASSERT_EQ("dev", loaded->GetSnapshotManager()->Branch()); } -TEST(TableTest, TestLoadedTableReadRetainsOperationSchemeMap) { +TEST(TableTest, TestLoadedTableReadRetainsLoadSchemeMap) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const std::string path = dir->Str() + "/db.db/tbl"; ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
created, CreateManagedTable(dir->GetFileSystem(), path, Identifier("db", "tbl"))); - FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db", "tbl"))); const std::map scheme_map = {{"custom", "local"}}; + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), {}, scheme_map); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db", "tbl"))); for (bool direct : {false, true}) { SCOPED_TRACE(direct); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr context, - ReadContextBuilder(path).WithFileSystemSchemeToIdentifierMap(scheme_map).Finish()); + TableLoadOptions context_builder_36442_load_options{path}; + std::map context_builder_36442_options; + ReadContextBuilder context_builder_36442; + context_builder_36442_load_options.fs_scheme_to_identifier_map = scheme_map; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_36442.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, - direct ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context))); + direct ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(context_builder_36442_options)); + return configured->NewRead(std::move(context)); + })() + : TableRead::Create(context_builder_36442_load_options, std::move(context), + context_builder_36442_options)); auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); ASSERT_EQ( @@ -2407,7 +2912,7 @@ TEST(TableTest, TestLoadedTableReadRetainsOperationSchemeMap) { } } -TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { +TEST(TableTest, TestReadRetainsLoadSchemeMapForMainAndFallback) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -2416,9 +2921,7 @@ TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
dev, CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl", "dev"), "dev")); const std::map table_map = {{"base", "local"}, {"shared", "mock_fs"}}; - const std::map read_map = {{"custom", "local"}, {"shared", "local"}}; - const std::map expected = { - {"base", "local"}, {"custom", "local"}, {"shared", "local"}}; + ASSERT_OK_AND_ASSIGN( std::shared_ptr table, FileStoreTable::Create(fs, dir->Str(), @@ -2426,13 +2929,17 @@ TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { Identifier("db", "tbl"), "", "main", table_map)); for (bool fallback : {false, true}) { SCOPED_TRACE(fallback); - ReadContextBuilder builder("unused_path"); - builder.WithFileSystemSchemeToIdentifierMap(read_map); + std::map builder_options; + ReadContextBuilder builder; + if (fallback) { - builder.AddOption(Options::SCAN_FALLBACK_BRANCH, "dev"); + builder_options[Options::SCAN_FALLBACK_BRANCH] = "dev"; } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_27, + table->Copy(builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + configured_table_27->NewRead(std::move(context))); std::vector reads; if (fallback) { auto* fallback_read = dynamic_cast(read.get()); @@ -2445,15 +2952,19 @@ TEST(TableTest, TestReadMergesSchemeMapsForMainAndFallback) { auto* append_read = dynamic_cast(data_read); ASSERT_NE(nullptr, append_read); const auto& internal = append_read->context_; - ASSERT_EQ(expected, + ASSERT_EQ(table_map, internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); ASSERT_EQ(fs, internal->GetCoreOptions().GetFileSystem()); ASSERT_EQ(dir->Str(), internal->GetPath()); } } - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder("unused_path").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + std::map context_builder_38424_options; + ReadContextBuilder context_builder_38424; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_38424.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_28, + table->Copy(context_builder_38424_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + configured_table_28->NewRead(std::move(context))); auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); ASSERT_EQ(table_map, @@ -2469,17 +2980,43 @@ TEST(TableTest, TestSystemTablePathRetainsSchemeMapForDependencyReads) { const std::map scheme_map = {{"file", "local"}}; for (bool reload_context : {false, true}) { SCOPED_TRACE(reload_context); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(dir->Str() + "$ro") - .WithFileSystemSchemeToIdentifierMap(scheme_map) - .Finish()); + TableLoadOptions context_builder_41670_load_options{dir->Str() + "$ro"}; + std::map context_builder_41670_options; + ReadContextBuilder context_builder_41670; + context_builder_41670_load_options.fs_scheme_to_identifier_map = scheme_map; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_41670.Finish()); std::unique_ptr read; if (reload_context) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(context.get())); - ASSERT_OK_AND_ASSIGN(context, ReadContextBuilder(dir->Str()).Finish()); - ASSERT_OK_AND_ASSIGN(read, table->NewRead(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + TableFactory::Create(context_builder_41670_load_options, + context_builder_41670_options, "read")); + TableLoadOptions context_builder_42682_load_options{dir->Str()}; + std::map context_builder_42682_options; + ReadContextBuilder context_builder_42682; + context_builder_41670_load_options.path = context_builder_42682_load_options.path; + context_builder_41670_load_options.file_system = + context_builder_42682_load_options.file_system; + context_builder_41670_load_options.fs_scheme_to_identifier_map = + context_builder_42682_load_options.fs_scheme_to_identifier_map; + context_builder_41670_load_options.table_schema_json = + context_builder_42682_load_options.table_schema_json; + context_builder_41670_load_options.catalog = context_builder_42682_load_options.catalog; + context_builder_41670_load_options.table_id = + context_builder_42682_load_options.table_id; + context_builder_41670_load_options.identifier.reset(); + if (context_builder_42682_load_options.identifier) { + context_builder_41670_load_options.identifier.emplace( + *context_builder_42682_load_options.identifier); + } + context_builder_41670_options = context_builder_42682_options; + ASSERT_OK_AND_ASSIGN(context, context_builder_42682.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_29, + table->Copy(context_builder_41670_options)); + ASSERT_OK_AND_ASSIGN(read, configured_table_29->NewRead(std::move(context))); } else { - ASSERT_OK_AND_ASSIGN(read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + read, TableRead::Create(context_builder_41670_load_options, std::move(context), + context_builder_41670_options)); } auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); @@ -2490,30 +3027,31 @@ TEST(TableTest, TestSystemTablePathRetainsSchemeMapForDependencyReads) { } } -TEST(TableTest, TestSystemTableReadMergesSchemeMapsWithoutReplacingFileSystem) { +TEST(TableTest, TestSystemTableReadRetainsLoadSchemeMapWithoutReplacingFileSystem) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); ASSERT_OK_AND_ASSIGN(std::shared_ptr
data_table, CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"))); const std::map table_map = {{"base", "local"}}; - const std::map read_map = {{"file", "local"}}; - const std::map expected = {{"base", "local"}, {"file", "local"}}; ASSERT_OK_AND_ASSIGN( std::shared_ptr table, SystemTableLoader::Load("ro", fs, dir->Str(), std::dynamic_pointer_cast(data_table->GetSchema()), {}, "tbl", table_map)); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr context, - ReadContextBuilder("unused_path").WithFileSystemSchemeToIdentifierMap(read_map).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + std::map context_builder_43111_options; + ReadContextBuilder context_builder_43111; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_43111.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_30, + table->Copy(context_builder_43111_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + configured_table_30->NewRead(std::move(context))); auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); const auto& internal = append_read->context_; ASSERT_EQ(fs, internal->GetCoreOptions().GetFileSystem()); ASSERT_EQ(dir->Str(), internal->GetPath()); - ASSERT_EQ(expected, internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); + ASSERT_EQ(table_map, internal->CreateDependencyCatalogContext()->fs_scheme_to_identifier_map); } TEST(TableTest, TestRealtimeFallbackOnlyAllowsMainBranch) { @@ -2537,13 +3075,20 @@ TEST(TableTest, TestRealtimeFallbackOnlyAllowsMainBranch) { for (bool realtime_enabled : {false, true}) { for (const std::string fallback : {"main", "dev"}) { SCOPED_TRACE(fallback + (realtime_enabled ? ":realtime" : ":disk")); - ReadContextBuilder builder(dir->Str()); - builder.SetOptions({{Options::SCAN_FALLBACK_BRANCH, fallback}}); + std::map builder_options; + ReadContextBuilder builder; + builder_options = {{Options::SCAN_FALLBACK_BRANCH, fallback}}; + if (realtime_enabled) { - builder.WithRealtimeContext(realtime).AddOption(Options::REALTIME_ENABLED, "true"); + builder_options[Options::REALTIME_ENABLED] = "true"; + builder.WithRealtimeContext(realtime); } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - auto read = table->NewRead(std::move(context)); + auto read = ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(builder_options)); + return configured->NewRead(std::move(context)); + })(); if (realtime_enabled && fallback != "main") { ASSERT_NOK_WITH_MSG(read, "real-time operations only support the main branch"); } else { @@ -2570,25 +3115,31 @@ TEST(TableTest, SystemTablePathUsesEffectiveBranchSchema) { const std::map options = path_branch ? std::map() : std::map{{Options::BRANCH, "dev"}}; + TableLoadOptions context_load_90{path}; + std::map context_options_90; + context_options_90 = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder().SetReadFieldNames({"extra"}).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr context, - ReadContextBuilder(path).SetOptions(options).SetReadFieldNames({"extra"}).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + std::unique_ptr read, + TableRead::Create(context_load_90, std::move(context), context_options_90)); auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); ASSERT_EQ("dev", append_read->context_->GetCoreOptions().GetBranch()); ASSERT_EQ(dev_schema->field_names(), append_read->context_->GetTableSchema()->FieldNames()); ASSERT_EQ((std::vector{"extra"}), append_read->context_->GetReadSchema()->field_names()); + TableLoadOptions scan_context_load_91{path}; + std::map scan_context_options_91; + scan_context_options_91 = options; ASSERT_OK_AND_ASSIGN( std::unique_ptr scan_context, - ScanContextBuilder(path) - .SetOptions(options) + ScanContextBuilder() .SetPredicate(PredicateBuilder::GreaterThan(1, "extra", FieldType::INT, Literal(0))) .Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_91, std::move(scan_context), + scan_context_options_91)); ASSERT_OK(scan->CreatePlan()); } ASSERT_NOK(SystemTableLoader::LoadFromPath(fs, dir->Str() + "$branch_dev$ro", @@ -2612,13 +3163,15 @@ TEST(TableTest, SystemTableMainAliasReturnsExistingSnapshots) { catalog.GetTable(Identifier("db", name))); ASSERT_EQ("db." + name, table->FullName()); ASSERT_EQ(table->FullName(), table->Uuid()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(table->GetRootPath()).Finish()); + ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(table->GetRootPath()).Finish()); + ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, @@ -2644,13 +3197,15 @@ TEST(TableTest, StaticFallbackReadUsesSuppliedMainSchema) { CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl", "dev"), "dev")); ASSERT_OK_AND_ASSIGN(std::string main_json, main->GetSchema()->GetJsonSchema()); ASSERT_OK(fs->Delete(dir->Str() + "/schema", true)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(dir->Str()) - .WithBranch("dev") - .AddOption(Options::SCAN_FALLBACK_BRANCH, "main") - .SetTableSchema(main_json) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, TableRead::Create(std::move(context))); + TableLoadOptions context_load_94{dir->Str()}; + std::map context_options_94; + context_options_94[Options::BRANCH] = "dev"; + context_options_94[Options::SCAN_FALLBACK_BRANCH] = "main"; + context_load_94.table_schema_json = main_json; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(context_load_94, std::move(context), context_options_94)); auto* fallback = dynamic_cast(read.get()); ASSERT_NE(nullptr, fallback); auto* fallback_read = dynamic_cast(fallback->fallback_table_.get()); @@ -2658,34 +3213,35 @@ TEST(TableTest, StaticFallbackReadUsesSuppliedMainSchema) { ASSERT_EQ(main->GetSchema()->FieldNames(), fallback_read->context_->GetTableSchema()->FieldNames()); ASSERT_EQ("main", fallback_read->context_->GetCoreOptions().GetBranch()); + TableLoadOptions missing_schema_load_95{dir->Str()}; + std::map missing_schema_options_95; + missing_schema_options_95[Options::BRANCH] = "dev"; + missing_schema_options_95[Options::SCAN_FALLBACK_BRANCH] = "main"; ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_schema, - ReadContextBuilder(dir->Str()) - .WithBranch("dev") - .AddOption(Options::SCAN_FALLBACK_BRANCH, "main") - .Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(missing_schema)), "fallback schema not found"); + ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(missing_schema_load_95, std::move(missing_schema), + missing_schema_options_95), + "fallback schema not found"); } -TEST(TableTest, LoadedFallbackReadIgnoresForeignContextSchema) { +TEST(TableTest, LoadedFallbackReadUsesBoundStorage) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); ASSERT_OK(CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"), "main")); ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev, CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl", "dev"), "dev")); - for (const std::string& context_path : {dir->Str() + "/other", dir->Str()}) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(context_path) - .WithFileSystem(std::make_shared( - std::map{}, "local", - std::map{})) - .WithBranch("dev") - .AddOption(Options::SCAN_FALLBACK_BRANCH, "main") - .SetTableSchema("invalid json") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, dev->NewRead(std::move(context))); - ASSERT_NE(nullptr, dynamic_cast(read.get())); - } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_32, + dev->Copy({{Options::SCAN_FALLBACK_BRANCH, "main"}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + configured_table_32->NewRead(std::move(context))); + auto* fallback = dynamic_cast(read.get()); + ASSERT_NE(nullptr, fallback); + auto* append = dynamic_cast(fallback->fallback_table_.get()); + ASSERT_NE(nullptr, append); + ASSERT_EQ(fs, append->context_->GetCoreOptions().GetFileSystem()); + ASSERT_EQ(dir->Str(), append->context_->GetPath()); } TEST(TableTest, CatalogMetadataDoesNotRequireDataFormatPlugin) { @@ -2709,7 +3265,8 @@ TEST(TableTest, CatalogMetadataDoesNotRequireDataFormatPlugin) { ASSERT_NE(nullptr, managed); ASSERT_OK(managed->GetSchemaManager()->Latest()); ASSERT_OK(managed->GetSnapshotManager()->LatestSnapshot()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder(path).Finish()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); ASSERT_NOK_WITH_MSG(table->NewRead(std::move(context)), "FileFormatFactory"); } @@ -2729,30 +3286,23 @@ TEST(TableTest, SystemTableFullNamesDistinguishDatabases) { } } -TEST(TableTest, LoadedQueryAuthRejectsForeignContextBeforePlanning) { +TEST(TableTest, LoadedQueryAuthKeepsBoundLocationAfterCatalogMoves) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, CreateQueryAuthTable(dir->Str())); ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); - FileSystemCatalog file_catalog(dir->GetFileSystem(), dir->Str(), {}); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - file_catalog.GetTable(Identifier("foo", "bar"))); - // Reject a foreign scope at operator construction; no data access is needed. - for (bool same_path : {false, true}) { - catalog->table_location = table->GetRootPath() + (same_path ? "" : "/other"); - const std::string error = same_path ? "context names table" : "context names root path"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - ScanContextBuilder(catalog->table_location) - .WithCatalog(catalog, Identifier("foo", "other")) - .Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan)), error); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - ReadContextBuilder(catalog->table_location) - .WithCatalog(catalog, Identifier("foo", "other")) - .Finish()); - ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read)), error); - } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); + catalog->table_location = table->GetRootPath() + "/other"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan)), "catalog keeps table"); + ASSERT_TRUE(catalog->auth_identifiers.empty()); + // Contexts cannot change the identity of a loaded handle. Restoring the matching location + // makes it usable again without reloading its schema or changing its storage binding. + catalog->table_location = table->GetRootPath(); + ASSERT_OK_AND_ASSIGN(scan, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, table->NewScan(std::move(scan))); + ASSERT_OK(table_scan->CreatePlan()); } TEST(TableTest, LoadedHandlesCheckNewAuthorizationSettings) { @@ -2771,12 +3321,18 @@ TEST(TableTest, LoadedHandlesCheckNewAuthorizationSettings) { SystemTableLoader::Load(kind, fs, dir->Str(), checked_pointer_cast(table->GetSchema()), {{Options::BRANCH, branch}})); + + std::map enabled_options_100; + enabled_options_100[Options::BRANCH] = branch; + enabled_options_100[Options::QUERY_AUTH_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr enabled, - ReadContextBuilder(dir->Str()) - .WithBranch(branch) - .AddOption(Options::QUERY_AUTH_ENABLED, "true") - .Finish()); - ASSERT_NOK_WITH_MSG(view->NewRead(std::move(enabled)), "query-auth.enabled"); + ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + view->Copy(enabled_options_100)); + return configured->NewRead(std::move(enabled)); + })(), + "query-auth.enabled"); views.push_back(std::move(view)); } ASSERT_OK_AND_ASSIGN( @@ -2787,19 +3343,37 @@ TEST(TableTest, LoadedHandlesCheckNewAuthorizationSettings) { const std::string schema_dir = branch == "main" ? "/schema" : "/branch/branch-dev/schema"; ASSERT_OK(fs->AtomicStore(dir->Str() + schema_dir + "/schema-1", json)); for (const auto& view : views) { + std::map context_options_101; + context_options_101[Options::BRANCH] = branch; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(dir->Str()).WithBranch(branch).Finish()); - ASSERT_NOK_WITH_MSG(view->NewRead(std::move(context)), "query-auth.enabled"); + ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + view->Copy(context_options_101)); + return configured->NewRead(std::move(context)); + })(), + "query-auth.enabled"); } - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - ReadContextBuilder(dir->Str()).WithBranch(branch).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr guarded, table->NewRead(std::move(read))); + + std::map read_options_102; + read_options_102[Options::BRANCH] = branch; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_35, + table->Copy(read_options_102)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr guarded, + configured_table_35->NewRead(std::move(read))); ASSERT_NE(nullptr, dynamic_cast(guarded.get())); ASSERT_NOK_WITH_MSG(guarded->CreateReader(std::shared_ptr()), "query-auth.enabled"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr scan, - ScanContextBuilder(dir->Str()).AddOption(Options::BRANCH, branch).Finish()); - ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan)), "without the catalog"); + + std::map scan_options_103; + scan_options_103[Options::BRANCH] = branch; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy(scan_options_103)); + return configured->NewScan(std::move(scan)); + })(), + "without the catalog"); } } @@ -2814,36 +3388,41 @@ TEST(TableTest, StaticReadOptimizedPreservesSuppliedSchemaAndMandatoryOptions) { arrow::field("extra", arrow::int32())}), {}, {}, {})); ASSERT_OK_AND_ASSIGN(std::string json, supplied->GetJsonSchema()); + TableLoadOptions context_load_104{dir->Str() + "$ro"}; + + context_load_104.table_schema_json = json; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ReadContextBuilder(dir->Str() + "$ro") - .SetTableSchema(json) - .SetReadFieldNames({"extra"}) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, TableRead::Create(std::move(context))); + ReadContextBuilder().SetReadFieldNames({"extra"}).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + TableRead::Create(context_load_104, std::move(context), {})); auto* append = dynamic_cast(read.get()); ASSERT_NE(nullptr, append); ASSERT_EQ((std::vector{"extra"}), append->context_->GetReadSchema()->field_names()); + TableLoadOptions scan_context_load_105{dir->Str() + "$ro"}; + std::map scan_context_options_105; + scan_context_load_105.table_schema_json = json; + scan_context_options_105[kReadOptimizedScanOption] = "false"; ASSERT_OK_AND_ASSIGN( std::unique_ptr scan_context, - ScanContextBuilder(dir->Str() + "$ro") - .SetTableSchema(json) + ScanContextBuilder() .SetPredicate(PredicateBuilder::GreaterThan(1, "extra", FieldType::INT, Literal(0))) - .AddOption(kReadOptimizedScanOption, "false") .Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_105, std::move(scan_context), + scan_context_options_105)); ASSERT_OK(scan->CreatePlan()); const std::string pk_path = dir->Str() + "/pk"; SchemaManager manager(dir->GetFileSystem(), pk_path); ASSERT_OK(manager.CreateTable(arrow::schema({arrow::field("id", arrow::int32(), false)}), {}, {"id"}, {{Options::BUCKET, "1"}})); - ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder(pk_path + "$ro") - .WithStreamingMode(true) - .AddOption(kReadOptimizedScanOption, "false") - .Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), - "read-optimized system table does not support streaming scan"); + TableLoadOptions scan_context_load_106{pk_path + "$ro"}; + std::map scan_context_options_106; + scan_context_options_106[kReadOptimizedScanOption] = "false"; + ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().WithStreamingMode(true).Finish()); + ASSERT_NOK_WITH_MSG( + TableScan::Create(scan_context_load_106, std::move(scan_context), scan_context_options_106), + "read-optimized system table does not support streaming scan"); } TEST(TableTest, StaticSystemTablePathPreservesCaseSensitiveBranch) { @@ -2853,15 +3432,17 @@ TEST(TableTest, StaticSystemTablePathPreservesCaseSensitiveBranch) { const std::string branch = "MAIN"; ASSERT_OK(CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"), branch)); const std::string path = dir->Str() + "$branch_MAIN$schemas"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path).Finish()); + TableLoadOptions scan_context_load_107{path}; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_107, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(path).Finish()); + TableLoadOptions read_context_load_108{path}; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_load_108, std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, ReadResultCollector::CollectResult(std::move(reader))); @@ -2878,14 +3459,17 @@ TEST(TableTest, StaticFallbackResolvesStorageFromFallbackOptions) { ASSERT_OK(main.CreateTable(schema, {}, {}, {{"test.endpoint", "fallback"}})); ASSERT_OK(dev.CreateTable(schema, {}, {}, {{"test.endpoint", "primary"}})); for (bool explicit_fs : {false, true}) { - ReadContextBuilder builder(dir->Str()); - builder.WithBranch("dev").AddOption(Options::SCAN_FALLBACK_BRANCH, "main"); + TableLoadOptions builder_load{dir->Str()}; + std::map builder_options; + ReadContextBuilder builder; + builder_options[Options::BRANCH] = "dev"; + builder_options[Options::SCAN_FALLBACK_BRANCH] = "main"; if (explicit_fs) { - builder.WithFileSystem(fs); + builder_load.file_system = fs; } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + TableRead::Create(builder_load, std::move(context), builder_options)); auto* fallback = dynamic_cast(read.get()); ASSERT_NE(nullptr, fallback); auto* append = dynamic_cast(fallback->fallback_table_.get()); @@ -2902,3 +3486,169 @@ TEST(TableTest, StaticFallbackResolvesStorageFromFallbackOptions) { } } // namespace paimon::test + +namespace paimon::test { + +TEST(TableTest, TestLoadOptionsValidateIdentityBeforeMetadataAccess) { + auto catalog = std::make_shared(); + TableLoadOptions load{"unused/table"}; + load.identifier.emplace("db", "tbl"); + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "read"), "non-null catalog"); + load.catalog = catalog; + load.identifier.reset(); + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "write"), "table identifier"); + load.identifier.emplace("db", "tbl", "dev"); + ASSERT_NOK(TableFactory::Create(load, {{Options::BRANCH, "other"}}, "read")); + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {{Options::BRANCH, "bad/branch"}}, "write"), + "branch"); + ASSERT_EQ(catalog->GetTableCalls(), 0u); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); +} + +TEST(TableTest, TestLoadOptionsRejectConflictingFileSystemResolution) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + TableLoadOptions load{dir->Str()}; + load.file_system = dir->GetFileSystem(); + load.fs_scheme_to_identifier_map = {{"file", "local"}}; + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "read"), "cannot be specified together"); +} + +TEST(TableTest, TestPreloadedSchemaIsIndependentOfReadProjection) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + TableSchema::Create(0, + arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("value", arrow::utf8())}), + {}, {}, {})); + TableLoadOptions load{dir->Str() + "/without-metadata"}; + load.file_system = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(load.table_schema_json, schema->GetJsonSchema()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ReadContextBuilder().SetReadFieldNames({"value"}).Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(load, {}, "read")); + ASSERT_EQ(table->GetSchema()->FieldNames(), (std::vector{"id", "value"})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + table->NewRead(std::move(context))); +} + +TEST(TableTest, TestLoadOptionsValidateExpectedCatalogUuid) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(0, arrow::schema({arrow::field("id", arrow::int32())}), {}, {}, {})); + auto catalog = std::make_shared(true, "current-uuid"); + catalog->SetTableSchema(schema); + catalog->SetFileSystem(dir->GetFileSystem()); + TableLoadOptions load{dir->Str()}; + load.catalog = catalog; + load.identifier.emplace("db", "tbl"); + load.table_id = "previous-uuid"; + ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "commit"), "does not match"); + load.table_id = "current-uuid"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(load, {}, "commit")); + ASSERT_EQ(table->CatalogUuid(), load.table_id); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + CommitContextBuilder("writer").Finish()); + ASSERT_NOK_WITH_MSG(([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->Copy({{Options::BRANCH, "dev"}})); + return configured->NewCommit(std::move(context)); + })(), + "branch contradicts"); +} + +TEST(TableTest, TestExplicitPathAndUuidRetainCatalogFileSystemWithRouting) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(0, arrow::schema({arrow::field("id", arrow::int32())}), {}, {}, {})); + ASSERT_OK_AND_ASSIGN(std::string schema_json, schema->GetJsonSchema()); + for (bool with_routing : {false, true}) { + std::map routes; + if (with_routing) { + routes["dependency"] = "local"; + } + auto scoped_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr catalog_table, + FileStoreTable::Create(scoped_fs, dir->Str(), schema, Identifier("db", "tbl"), + "table-uuid", "main", routes)); + auto catalog = std::make_shared(); + catalog->SetTable(catalog_table); + catalog->SetTableLocation(dir->Str()); + for (bool override_file_system : {false, true}) { + auto supplied_fs = std::make_shared(); + TableLoadOptions load{dir->Str()}; + load.catalog = catalog; + load.identifier.emplace("db", "tbl"); + load.table_id = "table-uuid"; + load.table_schema_json = schema_json; + if (override_file_system) { + load.file_system = supplied_fs; + } + for (const std::string operation : {"scan", "read", "write", "commit"}) { + SCOPED_TRACE(operation + (with_routing ? ":routing" : ":no-routing") + + (override_file_system ? ":override" : ":catalog")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + TableFactory::Create(load, {}, operation.c_str())); + ASSERT_EQ(override_file_system ? supplied_fs : scoped_fs, table->GetFileSystem()); + auto managed = std::dynamic_pointer_cast(table); + ASSERT_NE(managed, nullptr); + ASSERT_EQ(routes, managed->GetFileSystemSchemeToIdentifierMap()); + } + } + } +} + +TEST(TableTest, TestMissingSchemaErrorsDistinguishCatalogAndPathLoads) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + for (bool with_catalog : {false, true}) { + TableLoadOptions load{dir->Str() + "/db.db/missing-table"}; + if (with_catalog) { + load.catalog = std::make_shared( + dir->GetFileSystem(), dir->Str(), std::map{}); + load.identifier.emplace("db", "missing-table"); + } + for (const std::string operation : {"scan", "read"}) { + SCOPED_TRACE(operation + (with_catalog ? ":catalog" : ":path")); + auto result = TableFactory::Create(load, {}, operation.c_str()); + if (with_catalog) { + ASSERT_TRUE(result.status().IsNotExist()) << result.status().ToString(); + ASSERT_NOK_WITH_MSG(result, "missing-table"); + } else { + ASSERT_TRUE(result.status().IsInvalid()) << result.status().ToString(); + ASSERT_NOK_WITH_MSG(result, "not found latest schema in branch main"); + } + } + } +} + +} // namespace paimon::test + +namespace paimon::test { +TEST(TableTest, TestLoadedTableRetainsCatalogAfterStackCatalogIsDestroyed) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string path = dir->Str() + "/db.db/tbl"; + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
created, + CreateManagedTable(dir->GetFileSystem(), path, Identifier("db", "tbl"))); + std::shared_ptr
table; + const std::map catalog_options = {{"dependency.setting", "retained"}}; + { + FileSystemCatalog catalog(dir->GetFileSystem(), dir->Str(), catalog_options); + ASSERT_OK_AND_ASSIGN(table, catalog.GetTable(Identifier("db", "tbl"))); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(context))); + auto* append_read = dynamic_cast(read.get()); + ASSERT_NE(append_read, nullptr); + auto dependency = append_read->context_->CreateDependencyCatalogContext(); + ASSERT_EQ(dependency->root_path, dir->Str()); + ASSERT_EQ(dependency->options.at("dependency.setting"), "retained"); +} +} // namespace paimon::test diff --git a/src/paimon/core/utils/blob_view_lookup.cpp b/src/paimon/core/utils/blob_view_lookup.cpp index 30999fe62..0e02439b8 100644 --- a/src/paimon/core/utils/blob_view_lookup.cpp +++ b/src/paimon/core/utils/blob_view_lookup.cpp @@ -28,6 +28,7 @@ #include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/executor/future.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -154,27 +155,32 @@ Result BlobViewLookup::LoadTableDescriptorChu Catalog::Create(catalog_context->root_path, catalog_context->options, /*file_system=*/nullptr, catalog_context->fs_scheme_to_identifier_map)); PAIMON_ASSIGN_OR_RAISE(std::string table_path, catalog->GetTableLocation(identifier)); - ScanContextBuilder scan_builder(table_path); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); - scan_builder.SetGlobalIndexResult(global_index_result) - .WithMemoryPool(pool) - .WithCatalog(catalog, identifier); + scan_builder_load_options.catalog = catalog; + scan_builder_load_options.identifier.emplace(identifier); + scan_builder.SetGlobalIndexResult(global_index_result).WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - ReadContextBuilder read_builder(table_path); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; std::vector read_field_ids = field_ids; read_field_ids.push_back(SpecialFieldIds::ROW_ID); - read_builder.SetReadFieldIds(read_field_ids) - .AddOption(Options::BLOB_AS_DESCRIPTOR, "true") - .EnablePrefetch(true) - .WithMemoryPool(pool) - .WithCatalog(catalog, identifier); + read_builder_options[Options::BLOB_AS_DESCRIPTOR] = "true"; + read_builder_load_options.catalog = catalog; + read_builder_load_options.identifier.emplace(identifier); + read_builder.SetReadFieldIds(read_field_ids).EnablePrefetch(true).WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 49286be40..73a94a176 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -526,9 +526,13 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); + std::shared_ptr catalog = std::const_pointer_cast(weak_from_this().lock()); + if (!catalog) { + catalog = std::make_shared(*this); + } return FileStoreTable::Create(table_file_system, table_path, schema, identifier, uuid, branch.value_or(Identifier::kDefaultMainBranch), - fs_scheme_to_identifier_map_); + fs_scheme_to_identifier_map_, catalog); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 8067ef1b7..aa8ce0ec1 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -40,6 +40,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/table.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/catalog_options.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" @@ -890,16 +891,26 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsFactoryCatalog) { std::weak_ptr weak_catalog = catalog; ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("sys", "tables"))); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(table->GetRootPath()).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, + table->Copy(scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + configured_table_1->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(table->GetRootPath()).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_2, + table->Copy(read_context_builder_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + configured_table_2->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); catalog.reset(); table.reset(); + configured_table_1.reset(); + configured_table_2.reset(); ASSERT_FALSE(weak_catalog.expired()); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); ASSERT_TRUE(BatchReader::IsEofBatch(batch)); @@ -1441,8 +1452,8 @@ TEST_F(RestCatalogTest, FormatTableUsesBoundFileSystemForStorageAccess) { ASSERT_NE(catalog->GetFileSystem(), table->GetFileSystem()); } ASSERT_TRUE(TokenRequests().empty()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - ScanContextBuilder(dir->Str()).Finish()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_TRUE(plan->Splits().empty()); @@ -1824,10 +1835,13 @@ TEST_F(RestCatalogTest, FileStoreCommitIsBuiltFromTheCatalog) { requests_before = state_->get_table_requests; } std::shared_ptr shared_catalog(std::move(rest_catalog)); - CommitContextBuilder builder(dir->Str(), "commit-user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.WithCatalog(shared_catalog, Identifier("db1", "t1")).Finish()); - ASSERT_OK(FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + CommitContextBuilder builder("commit-user"); + builder_load_options.catalog = shared_catalog; + builder_load_options.identifier.emplace(Identifier("db1", "t1")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK(FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); { std::lock_guard lock(state_->mutex); ASSERT_GT(state_->get_table_requests, requests_before); @@ -1889,16 +1903,19 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { ASSERT_EQ(1, state_->get_table_requests); } + TableLoadOptions scan_context_load_2{dir->Str()}; + + scan_context_load_2.catalog = catalog; + scan_context_load_2.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan_context, - ScanContextBuilder(dir->Str()) - .WithCatalog(catalog, identifier) + ScanContextBuilder() .SetReadFieldNames({"f1"}) .SetPredicate(PredicateBuilder::GreaterThan(/*field_index=*/0, "f0", FieldType::INT, Literal(static_cast(0)))) .Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_load_2, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); { std::lock_guard lock(state_->mutex); @@ -1912,11 +1929,13 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { sent.GetSelect()); } + TableLoadOptions read_context_load_3{dir->Str()}; + std::map read_context_options_3; + read_context_options_3[Options::QUERY_AUTH_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr read_context, - ReadContextBuilder(dir->Str()).AddOption(Options::QUERY_AUTH_ENABLED, "true").Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + std::unique_ptr read, + TableRead::Create(read_context_load_3, std::move(read_context), read_context_options_3)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr chunks, ReadResultCollector::CollectResult(std::move(reader))); @@ -1927,61 +1946,86 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { ASSERT_EQ("***", checked_pointer_cast(rows->GetFieldByName("f1"))->GetString(0)); + TableLoadOptions disabling_read_context_load_4{dir->Str()}; + std::map disabling_read_context_options_4; + disabling_read_context_load_4.catalog = catalog; + disabling_read_context_load_4.identifier.emplace(identifier); + disabling_read_context_options_4[Options::QUERY_AUTH_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::unique_ptr disabling_read_context, - ReadContextBuilder(dir->Str()) - .WithCatalog(catalog, identifier) - .AddOption(Options::QUERY_AUTH_ENABLED, "false") - .Finish()); + ReadContextBuilder().Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(std::move(disabling_read_context)), + TableRead::Create(disabling_read_context_load_4, std::move(disabling_read_context), + disabling_read_context_options_4), "Invalid: Option 'query-auth.enabled' is not allowed to be disabled dynamically"); + TableLoadOptions files_scan_context_load_5{dir->Str() + "$files"}; + + files_scan_context_load_5.catalog = catalog; + files_scan_context_load_5.identifier.emplace(Identifier("db1", "t1$files")); ASSERT_OK_AND_ASSIGN(std::unique_ptr files_scan_context, - ScanContextBuilder(dir->Str() + "$files") - .WithCatalog(catalog, Identifier("db1", "t1$files")) - .Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(files_scan_context)), - "NotImplemented: System table 'files' is not supported on a table with " - "'query-auth.enabled=true'"); + ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableScan::Create(files_scan_context_load_5, std::move(files_scan_context), {}), + "NotImplemented: System table 'files' is not supported on a table with " + "'query-auth.enabled=true'"); + TableLoadOptions options_scan_context_load_6{dir->Str() + "$options"}; + + options_scan_context_load_6.catalog = catalog; + options_scan_context_load_6.identifier.emplace(Identifier("db1", "t1$options")); ASSERT_OK_AND_ASSIGN(std::unique_ptr options_scan_context, - ScanContextBuilder(dir->Str() + "$options") - .WithCatalog(catalog, Identifier("db1", "t1$options")) - .Finish()); - ASSERT_OK(TableScan::Create(std::move(options_scan_context)).status()); + ScanContextBuilder().Finish()); + ASSERT_OK(TableScan::Create(options_scan_context_load_6, std::move(options_scan_context), {}) + .status()); // A global system table identifier must not skip the catalog policy of a data table, // including a table whose published schema does not enable query auth. const Identifier system_identifier("sys", "tables"); for (const std::string& suffix : {std::string(), std::string("$files")}) { SCOPED_TRACE(suffix); + TableLoadOptions mismatched_scan_context_load_7{dir->Str() + suffix}; + + mismatched_scan_context_load_7.catalog = catalog; + mismatched_scan_context_load_7.identifier.emplace(system_identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr mismatched_scan_context, - ScanContextBuilder(dir->Str() + suffix) - .WithCatalog(catalog, system_identifier) - .Finish()); - Status scan_status = TableScan::Create(std::move(mismatched_scan_context)).status(); + ScanContextBuilder().Finish()); + Status scan_status = TableScan::Create(mismatched_scan_context_load_7, + std::move(mismatched_scan_context), {}) + .status(); + TableLoadOptions mismatched_read_context_load_8{dir->Str() + suffix}; + + mismatched_read_context_load_8.catalog = catalog; + mismatched_read_context_load_8.identifier.emplace(system_identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr mismatched_read_context, - ReadContextBuilder(dir->Str() + suffix) - .WithCatalog(catalog, system_identifier) - .Finish()); - Status read_status = TableRead::Create(std::move(mismatched_read_context)).status(); + ReadContextBuilder().Finish()); + Status read_status = TableRead::Create(mismatched_read_context_load_8, + std::move(mismatched_read_context), {}) + .status(); ASSERT_NOK_WITH_MSG(scan_status, "names a global system table, but the scan targets"); ASSERT_NOK_WITH_MSG(read_status, "names a global system table, but the read targets"); } const std::string global_path = PathUtil::JoinPath(dir->Str(), "sys/catalog_options"); const Identifier global_identifier("sys", "catalog_options"); + TableLoadOptions global_scan_context_load_9{global_path}; + std::map global_scan_context_options_9; + global_scan_context_load_9.catalog = catalog; + global_scan_context_load_9.identifier.emplace(global_identifier); + global_scan_context_options_9[CatalogOptionsSystemTable::kEnabledOption] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr global_scan_context, - ScanContextBuilder(global_path) - .WithCatalog(catalog, global_identifier) - .AddOption(CatalogOptionsSystemTable::kEnabledOption, "true") - .Finish()); - ASSERT_OK(TableScan::Create(std::move(global_scan_context)).status()); + ScanContextBuilder().Finish()); + ASSERT_OK(TableScan::Create(global_scan_context_load_9, std::move(global_scan_context), + global_scan_context_options_9) + .status()); + TableLoadOptions global_read_context_load_10{global_path}; + std::map global_read_context_options_10; + global_read_context_load_10.catalog = catalog; + global_read_context_load_10.identifier.emplace(global_identifier); + global_read_context_options_10[CatalogOptionsSystemTable::kEnabledOption] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr global_read_context, - ReadContextBuilder(global_path) - .WithCatalog(catalog, global_identifier) - .AddOption(CatalogOptionsSystemTable::kEnabledOption, "true") - .Finish()); - ASSERT_OK(TableRead::Create(std::move(global_read_context)).status()); + ReadContextBuilder().Finish()); + ASSERT_OK(TableRead::Create(global_read_context_load_10, std::move(global_read_context), + global_read_context_options_10) + .status()); } TEST_F(RestCatalogTest, GlobalPartitionsProtectsQueryAuthPartitionValues) { @@ -2079,12 +2123,13 @@ TEST_F(RestCatalogTest, QueryAuthRejectsColumnsReplacedOnlyInTheCatalog) { std::shared_ptr catalog(std::move(rest_catalog)); auto new_scan = [&](const std::vector& fields) -> Result> { + TableLoadOptions context_load_11{dir->Str()}; + + context_load_11.catalog = catalog; + context_load_11.identifier.emplace(Identifier("db1", "t1")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - ScanContextBuilder(dir->Str()) - .WithCatalog(catalog, Identifier("db1", "t1")) - .SetReadFieldNames(fields) - .Finish()); - return TableScan::Create(std::move(context)); + ScanContextBuilder().SetReadFieldNames(fields).Finish()); + return TableScan::Create(context_load_11, std::move(context), {}); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, new_scan({"f1"})); ASSERT_OK(scan->CreatePlan().status()); @@ -2163,13 +2208,15 @@ TEST_F(RestCatalogTest, ScanOfABranchFollowsTheQueryAuthOfThatBranch) { } std::shared_ptr catalog(std::move(rest_catalog)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(dir->Str()) - .WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_load_12{dir->Str()}; + std::map scan_context_options_12; + scan_context_load_12.catalog = catalog; + scan_context_load_12.identifier.emplace(Identifier("db1", "t1")); + scan_context_options_12[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(scan_context_load_12, std::move(scan_context), scan_context_options_12)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); { std::lock_guard lock(state_->mutex); @@ -2178,14 +2225,15 @@ TEST_F(RestCatalogTest, ScanOfABranchFollowsTheQueryAuthOfThatBranch) { state_->auth_requests); } - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(dir->Str()) - .WithCatalog(catalog, Identifier("db1", "t1")) - .AddOption(Options::BRANCH, "dev") - .Finish()); - ASSERT_EQ(Identifier("db1", "t1$branch_dev"), read_context->GetIdentifier().value()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableLoadOptions read_context_load_13{dir->Str()}; + std::map read_context_options_13; + read_context_load_13.catalog = catalog; + read_context_load_13.identifier.emplace(Identifier("db1", "t1")); + read_context_options_13[Options::BRANCH] = "dev"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(read_context_load_13, std::move(read_context), read_context_options_13)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr chunks, ReadResultCollector::CollectResult(std::move(reader))); @@ -2228,23 +2276,29 @@ TEST_F(RestCatalogTest, ReadRejectsAFallbackBranchProtectedOnlyByTheCatalog) { } } std::shared_ptr catalog(std::move(rest_catalog)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(dir->Str()) - .WithCatalog(catalog, Identifier("db1", "t1")) - .WithBranch("dev") - .AddOption(Options::SCAN_FALLBACK_BRANCH, "main") - .Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), - "Cannot read fallback branch 'main', which enables 'query-auth.enabled'"); + TableLoadOptions read_context_load_14{dir->Str()}; + std::map read_context_options_14; + read_context_load_14.catalog = catalog; + read_context_load_14.identifier.emplace(Identifier("db1", "t1")); + read_context_options_14[Options::BRANCH] = "dev"; + read_context_options_14[Options::SCAN_FALLBACK_BRANCH] = "main"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_context_load_14, std::move(read_context), read_context_options_14), + "Cannot read fallback branch 'main', which enables 'query-auth.enabled'"); for (const auto& [branch, message] : std::vector>{ {"MAIN", "a catalog names branch 'MAIN' as it names the main branch"}, {"dev$options", "a branch a catalog addresses cannot contain '$'"}}) { - ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder(dir->Str()) - .WithCatalog(catalog, Identifier("db1", "t1")) - .WithBranch("dev") - .AddOption(Options::SCAN_FALLBACK_BRANCH, branch) - .Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), message); + TableLoadOptions read_context_load_15{dir->Str()}; + std::map read_context_options_15; + read_context_load_15.catalog = catalog; + read_context_load_15.identifier.emplace(Identifier("db1", "t1")); + read_context_options_15[Options::BRANCH] = "dev"; + read_context_options_15[Options::SCAN_FALLBACK_BRANCH] = branch; + ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_load_15, std::move(read_context), + read_context_options_15), + message); } } @@ -2555,10 +2609,13 @@ TEST_P(RestCatalogCommitRecoveryTest, CommitAcceptedThenLostRecoversAfterRestart std::shared_ptr catalog(std::move(rest_catalog)); auto create_commit = [&]() -> Result> { - CommitContextBuilder builder(dir->Str(), "commit-user"); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.WithCatalog(catalog, identifier).Finish()); - return FileStoreCommit::Create(std::move(context)); + TableLoadOptions builder_load_options{dir->Str()}; + std::map builder_options; + CommitContextBuilder builder("commit-user"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(identifier); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + return FileStoreCommit::Create(builder_load_options, std::move(context), builder_options); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, create_commit()); @@ -2730,16 +2787,20 @@ class RestCatalogCommitLeaseTest : public RestCatalogTest { Result> CreateCommit( const std::shared_ptr& catalog, const std::optional& table_id, const std::map& options) { - CommitContextBuilder builder(dir_->Str(), "commit-user"); - builder.WithCatalog(catalog, Identifier("db1", "t1")); + TableLoadOptions builder_load_options{dir_->Str()}; + std::map builder_options; + CommitContextBuilder builder("commit-user"); + builder_load_options.catalog = catalog; + builder_load_options.identifier.emplace(Identifier("db1", "t1")); + if (table_id) { - builder.WithTableId(table_id.value()); + builder_load_options.table_id = table_id.value(); } for (const auto& [key, value] : options) { - builder.AddOption(key, value); + builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(std::move(context)); + return FileStoreCommit::Create(builder_load_options, std::move(context), builder_options); } std::vector TableEvents() { diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index efc2caeba..34deaad56 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -114,6 +114,9 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { Result> GetTable(const Identifier& identifier) const override { ++get_table_calls_; + if (table_ != nullptr) { + return table_; + } // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location, // even when `SetTableLocation()` set one. @@ -136,6 +139,13 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { table_schema_ = table_schema; } + void SetTable(const std::shared_ptr
& table) { + table_ = table; + table_uuid_ = table->CatalogUuid().value_or(""); + table_schema_ = table->GetSchema(); + table_file_system_ = table->GetFileSystem(); + } + Result> LoadTableSchema(const Identifier& identifier) const override { load_table_schema_identifiers_.push_back(identifier); if (table_schema_ == nullptr) { @@ -179,6 +189,10 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { return load_snapshot_identifiers_.size(); } + void SetTableUuid(const std::string& table_uuid) { + table_uuid_ = table_uuid; + } + void CheckTableUuid() { check_table_uuid_ = true; } @@ -313,6 +327,7 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { Result answer_; std::string table_uuid_; + std::shared_ptr
table_; std::shared_ptr table_schema_; std::optional table_location_; std::shared_ptr file_system_; diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index bc87fcf1b..87bf02654 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -32,6 +32,7 @@ #include "fmt/format.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" @@ -90,23 +91,30 @@ class TestHelper { PAIMON_ASSIGN_OR_RAISE(auto file_system, FileSystemFactory::Get(file_system_identifier, table_path, options)); std::string commit_user = "commit_user"; - WriteContextBuilder context_builder(table_path, commit_user); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); if (!temp_directory.empty()) { context_builder.WithTempDirectory(temp_directory); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithStreamingMode(is_streaming_mode) - .WithWriteId(12345) - .Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); - CommitContextBuilder commit_context_builder(table_path, commit_user); + context_builder_options = options; PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).IgnoreEmptyCommit(false).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + std::unique_ptr write_context, + context_builder.WithStreamingMode(is_streaming_mode).WithWriteId(12345).Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(commit_user); + commit_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); return std::unique_ptr(new TestHelper(std::move(file_system), std::move(write), std::move(commit), commit_user, table_path, options)); @@ -156,16 +164,20 @@ class TestHelper { Result>> NewScan(StartupMode startup_mode, std::optional snapshot_id, bool is_streaming = true) { - ScanContextBuilder scan_context_builder(table_path_); - scan_context_builder.WithStreamingMode(is_streaming) - .SetOptions(options_) - .AddOption(Options::SCAN_MODE, startup_mode.ToString()); + TableLoadOptions scan_context_builder_load_options{table_path_}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options_; + scan_context_builder_options[Options::SCAN_MODE] = startup_mode.ToString(); + scan_context_builder.WithStreamingMode(is_streaming); if (snapshot_id) { - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, - std::to_string(snapshot_id.value())); + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = + std::to_string(snapshot_id.value()); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(scan_, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + scan_, TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); return Scan(); } @@ -276,14 +288,19 @@ class TestHelper { Result> ReadResult( const std::vector>& splits, std::unique_ptr<::ArrowSchema> read_schema) { - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options_); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options_; + if (read_schema != nullptr) { read_context_builder.SetReadSchema(std::move(read_schema)); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr collected, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -302,11 +319,16 @@ class TestHelper { Result ReadAndCheckResult(const std::shared_ptr& data_type, const std::vector>& splits, const std::string& expected_result) { - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options_); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index ae5dcf086..6b420a6e9 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -24,6 +24,7 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" @@ -415,10 +416,16 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMap auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{ + PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1038,10 +1045,16 @@ TEST_P(AppendCompactionInteTest, TestAppendTableCompactionDictionaryPassthrough) auto predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"s", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); - ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); - read_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions read_context_builder_load_options{ + PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto filtered, ReadResultCollector::CollectResult(batch_reader.get())); auto expected = std::make_shared( diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 1bd320e49..f2031fdbe 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -40,6 +40,7 @@ #include "arrow/ipc/json_simple.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" @@ -141,11 +142,15 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter const std::vector& write_cols, const std::vector>& write_arrays) const { // write - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); for (const auto& write_array : write_arrays) { ArrowArray c_array; @@ -176,11 +181,15 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -267,14 +276,18 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter Result> ScanTable(const std::string& table_path, const std::shared_ptr& predicate = nullptr, const std::vector& row_ranges = {}) const { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); return result_plan; } @@ -285,15 +298,19 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter const std::shared_ptr& plan, const std::shared_ptr& predicate = nullptr, const std::map& options = {}) const { auto splits = plan->Splits(); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); read_context_builder.EnableLateMaterializing(false); if (!options.empty()) { - read_context_builder.SetOptions(options); + read_context_builder_options = options; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4896,10 +4913,15 @@ TEST_P(BlobTableInteTest, TestReadArrayBlobTableFromJava) { const std::vector read_fields = {"id", "array_payloads"}; for (int64_t snapshot_id : {1, 2, 3}) { - ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); size_t array_layer_count = 0; @@ -4957,10 +4979,15 @@ TEST_P(BlobTableInteTest, TestReadMapBlobTableFromJava) { "large_decimal_payloads"}; for (int64_t snapshot_id : {1, 3}) { - ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); size_t string_layer_count = 0; diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index ec418d2c0..4a32b7d48 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -42,6 +42,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -204,11 +205,16 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { auto dir = UniqueTestDirectory::Create(); std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -219,11 +225,16 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1"))); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -234,11 +245,16 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "f1=10/bucket-1/data-7a912f84-04b7-4bbb-8dc6-53f4a292ea25-0.parquet"))); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); } @@ -248,11 +264,16 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath(table_path, "snapshot/snapshot-3"))); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = clean_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.SetOptions(clean_options).IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 1); } @@ -282,12 +303,16 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { CreateTestTable(dir->Str(), /*db_name=*/"foo", /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -303,18 +328,24 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + commit_context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + commit_context_builder_options[Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "2") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true") + commit_context_builder + .IgnoreEmptyCommit(false) .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0, 10)); ASSERT_OK_AND_ASSIGN( @@ -430,12 +461,17 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -451,19 +487,25 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; + commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; + commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; + commit_context_builder_options[Options::MANIFEST_MERGE_MIN_COUNT] = "2"; + commit_context_builder_options[Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder + + .IgnoreEmptyCommit(false) + .Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "2") - .AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1") - .AddOption(Options::SNAPSHOT_TIME_RETAINED, "1ms") - .AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2") - .AddOption(Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true") - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -588,12 +630,16 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, commit_user); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}, {"f2", "1"}}, 0)); @@ -609,13 +655,16 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(table_path, commit_user); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(commit_user); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -702,12 +751,16 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, commit_user); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}, {"f2", "1"}}, 0)); @@ -723,13 +776,16 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(table_path, commit_user); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(commit_user); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .IgnoreEmptyCommit(false) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( @@ -812,12 +868,16 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, commit_user); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder(commit_user); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - std::string root_path = write_context->GetRootPath(); + context_builder.WithStreamingMode(true).Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -833,14 +893,20 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - CommitContextBuilder commit_context_builder(root_path, commit_user); + TableLoadOptions commit_context_builder_load_options{root_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder(commit_user); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_options[Options::BUCKET] = "2"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .WithFileSystem(file_system_) - .AddOption(Options::BUCKET, "2") + commit_context_builder + .IgnoreEmptyCommit(false) .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); ASSERT_OK_AND_ASSIGN( diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 79e16a6f2..5b8f333f0 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -24,6 +24,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -111,14 +112,18 @@ class DataEvolutionTableTest : public ::testing::Test, const std::vector>& write_arrays, const std::map& write_options = {}) const { // write - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithWriteSchema(write_cols); for (const auto& [key, value] : write_options) { - write_builder.AddOption(key, value); + write_builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); for (const auto& write_array : write_arrays) { ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); @@ -165,11 +170,15 @@ class DataEvolutionTableTest : public ::testing::Test, Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -308,29 +317,38 @@ class DataEvolutionTableTest : public ::testing::Test, const std::string& table_path, const std::vector& read_schema, int32_t limit, const std::shared_ptr& predicate = nullptr, const std::vector& row_ranges = {}, bool enable_predicate_filter = false) const { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(limit); scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { scan_context_builder.SetGlobalIndexResult( BitmapGlobalIndexResult::FromRanges(row_ranges)); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result_plan, table_scan->CreatePlan()); LimitScanResult result; result.splits = result_plan->Splits(); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadFieldNames(read_schema) .SetPredicate(predicate) .EnablePredicateFilter(enable_predicate_filter); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(result.splits)); PAIMON_ASSIGN_OR_RAISE(result.rows, @@ -343,11 +361,16 @@ class DataEvolutionTableTest : public ::testing::Test, /// the row count derived from them. Reading the splits back is ScanAndReadWithLimit's job, /// which verifies the returned rows after their reader has been destroyed. Result>> PlanSplits(const std::string& table_path) const { - ScanContextBuilder scan_context_builder(table_path); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); return plan->Splits(); } @@ -416,11 +439,15 @@ class DataEvolutionTableTest : public ::testing::Test, const std::string& table_path, const std::vector>& commit_msgs, std::optional row_id_check_from_snapshot) const { - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); file_store_commit->RowIdCheckConflict(row_id_check_from_snapshot); return file_store_commit->Commit(commit_msgs); } @@ -478,11 +505,16 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::string anchor, PlannedAnchorFileName(table_path)); ASSERT_OK(CommitDeletionVectors(table_path, base_msgs[0], {{anchor, deleted}})); } - ScanContextBuilder scan_builder(table_path); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; scan_builder.SetGlobalIndexResult(std::make_shared( RoaringBitmap64::From(candidates), std::vector(scores))); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_builder, scan_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto scan, TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto scored_split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -513,12 +545,16 @@ class DataEvolutionTableTest : public ::testing::Test, std::shared_ptr bound_predicate, PredicateUtils::CreatePickedFieldFilter(filter_case.predicate, {{"f0", project_row_id ? 3 : 2}})); - ReadContextBuilder read_builder(table_path); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; read_builder.SetReadFieldNames(read_fields) .SetPredicate(bound_predicate) .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(read_builder_load_options, + std::move(read_context), + read_builder_options)); // Reuse the same TableRead with and without scores: the internal row-id // projection must not affect a subsequent split's output schema. for (bool with_scores : {true, false}) { @@ -568,14 +604,19 @@ class DataEvolutionTableTest : public ::testing::Test, bool check_scan_plan_when_empty_result = true, bool apply_predicate_to_scan = true) const { // scan - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetPredicate(apply_predicate_to_scan ? predicate : nullptr); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder)); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder, + scan_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (!expected_array && check_scan_plan_when_empty_result) { if (!result_plan->Splits().empty()) { @@ -585,11 +626,15 @@ class DataEvolutionTableTest : public ::testing::Test, // read auto splits = result_plan->Splits(); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -634,14 +679,19 @@ class DataEvolutionTableTest : public ::testing::Test, const std::vector>& expected_first_row_ids, const std::vector& expected_row_counts) { ASSERT_EQ(expected_first_row_ids.size(), expected_row_counts.size()); - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); const auto& result_splits = result_plan->Splits(); if (expected_first_row_ids.empty()) { @@ -666,14 +716,15 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_EQ(result_row_counts, expected_row_counts); } - Result> FinishScanContext(ScanContextBuilder& builder) const { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& builder_options) const { if (EnableSnapshotLiveManifestCache()) { if (!snapshot_live_manifest_cache_) { snapshot_live_manifest_cache_ = std::make_shared(/*max_weight=*/64 * 1024 * 1024); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(snapshot_live_manifest_cache_); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(snapshot_live_manifest_cache_); } return builder.Finish(); } @@ -1098,16 +1149,25 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2019,11 +2079,14 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { } auto selected_gap_expected = checked_pointer_cast( arrow::Concatenate(selected_gap_chunks).ValueOrDie()); - ReadContextBuilder read_builder(table_path); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; read_builder.SetReadFieldNames({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); const std::vector, std::shared_ptr>> gap_cases = {{gap_split, gap_expected}, {std::make_shared(gap_split, selected_ranges), @@ -2084,12 +2147,16 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { {std::vector{"id", "embedding", "_ROW_ID", "_SEQUENCE_NUMBER"}, std::vector{"id", "_ROW_ID", "_SEQUENCE_NUMBER"}, std::vector{"_ROW_ID", "_SEQUENCE_NUMBER"}}) { - ReadContextBuilder sequence_read_builder(table_path); + TableLoadOptions sequence_read_builder_load_options{table_path}; + std::map sequence_read_builder_options; + ReadContextBuilder sequence_read_builder; sequence_read_builder.SetReadFieldNames(projection); ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_context, sequence_read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_read, - TableRead::Create(std::move(sequence_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr sequence_read, + TableRead::Create(sequence_read_builder_load_options, std::move(sequence_context), + sequence_read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_reader, sequence_read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr sequence_result, @@ -2197,16 +2264,25 @@ TEST_P(DedicatedVectorTableTest, TestDedicatedVectorStore) { ASSERT_EQ(group[0]->row_count, group[1]->row_count); } - WriteContextBuilder partial_builder(table_path, "partial"); + TableLoadOptions partial_builder_load_options{table_path}; + std::map partial_builder_options; + WriteContextBuilder partial_builder("partial"); partial_builder.WithWriteSchema({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr partial_context, partial_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(partial_context)).status(), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(partial_builder_load_options, + std::move(partial_context), partial_builder_options) + .status(), "partial writes"); - WriteContextBuilder invalid_builder(table_path, "invalid"); - invalid_builder.AddOption(Options::DATA_EVOLUTION_ENABLED, "false"); + TableLoadOptions invalid_builder_load_options{table_path}; + std::map invalid_builder_options; + WriteContextBuilder invalid_builder("invalid"); + invalid_builder_options[Options::DATA_EVOLUTION_ENABLED] = "false"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_context, invalid_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(invalid_context)).status(), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(invalid_builder_load_options, + std::move(invalid_context), invalid_builder_options) + .status(), "requires row-tracking.enabled and data-evolution.enabled"); } @@ -3972,12 +4048,16 @@ TEST_P(DataEvolutionDvTableTest, TestReadWithDeletionVectorsOnEveryRowRangeGroup // a count query answers from that metadata alone, never reading a row, so the deletion // vectors have to reach it too: without them it reports the 8 rows the files hold - ReadContextBuilder count_context_builder(table_path); + TableLoadOptions count_context_builder_load_options{table_path}; + std::map count_context_builder_options; + ReadContextBuilder count_context_builder; count_context_builder.SetReadFieldNames(schema->field_names()); ASSERT_OK_AND_ASSIGN(std::unique_ptr count_context, count_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr count_table_read, - TableRead::Create(std::move(count_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr count_table_read, + TableRead::Create(count_context_builder_load_options, std::move(count_context), + count_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr count_reader, count_table_read->CreateCountReader(planned_splits)); ASSERT_OK_AND_ASSIGN(int64_t counted_rows, count_reader->CountRows()); diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 25a9eceb3..43cfdcbc4 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -26,6 +26,7 @@ #include "fmt/format.h" #include "fmt/ranges.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_factory.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_options.h" @@ -224,12 +225,17 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::vector& write_cols, const std::shared_ptr& write_array) const { // write - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithWriteSchema(write_cols); - write_builder.WithFileSystem(fs_); + write_builder_load_options.file_system = fs_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); ArrowArray c_array; EXPECT_TRUE(arrow::ExportArray(*write_array, &c_array).ok()); auto record_batch = std::make_unique( @@ -252,23 +258,33 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit - CommitContextBuilder commit_builder(table_path, "commit_user_1"); - commit_builder.WithFileSystem(fs_); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); + commit_builder_load_options.file_system = fs_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } Result> ScanData( const std::string& table_path, const std::vector>& partition_filters) const { - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetPartitionFilter(partition_filters); - scan_context_builder.WithFileSystem(fs_); + scan_context_builder_load_options.file_system = fs_; + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); EXPECT_EQ(result_plan->Splits().size(), 1); return std::dynamic_pointer_cast(result_plan->Splits()[0]); @@ -1169,16 +1185,19 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::map& options = {}, const std::shared_ptr& index_result = nullptr, const std::shared_ptr& cache = nullptr) const { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPredicate(predicate) - .SetOptions(options) - .SetGlobalIndexResult(index_result) - .WithFileSystem(fs_); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_load_options.file_system = fs_; + scan_context_builder.SetPredicate(predicate).SetGlobalIndexResult(index_result); if (cache) { scan_context_builder.WithCache(cache); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); return result_plan; } @@ -1210,14 +1229,18 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::shared_ptr& predicate, const std::shared_ptr& result_plan) const { auto splits = result_plan->Splits(); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_load_options.file_system = fs_; read_context_builder.SetReadFieldNames(read_schema) .SetPredicate(predicate) - .WithFileSystem(fs_) .EnableLateMaterializing(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1601,13 +1624,17 @@ TEST_P(GlobalIndexTest, TestWriteIndexDecodesNestedOrcDictionaries) { ScanData(table_path, /*partition_filters=*/{})); std::shared_ptr indexed_split = std::make_shared(split, std::vector{Range(0, 3)}); - ReadContextBuilder read_builder(table_path); - read_builder.SetOptions(options).WithFileSystem(fs_).SetReadFieldNames( - {"key", "details", "attributes", "plain_details", "plain_attributes", - SpecialFields::RowId().Name()}); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options; + read_builder_load_options.file_system = fs_; + read_builder.SetReadFieldNames({"key", "details", "attributes", "plain_details", + "plain_attributes", SpecialFields::RowId().Name()}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(indexed_split)); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp index b9b1b99ca..6adf94267 100644 --- a/test/inte/nested_column_pruning_inte_test.cpp +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -29,6 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/path_util.h" @@ -92,25 +93,34 @@ class NestedColumnPruningInteTest : public ::testing::Test, const std::shared_ptr& expected_schema, const std::string& expected_json, const std::shared_ptr& predicate = nullptr) const { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + if (predicate) { scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*expected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetReadSchema(std::move(c_schema)); if (predicate) { read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -350,10 +360,14 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsAllNonExistent) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -412,10 +426,14 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithNonExistentField) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -491,10 +509,15 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithTypeMismatchShouldFa helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(data_splits), "PruneDataType nested field type mismatch for 'a': read string vs " @@ -581,10 +604,15 @@ TEST_P(NestedColumnPruningInteTest, auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); @@ -766,10 +794,15 @@ TEST_P(NestedColumnPruningInteTest, PruneNestedStructWithSpecialFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1178,10 +1211,16 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); - WriteContextBuilder write_context_builder(table_path_, "commit_user_1"); + TableLoadOptions write_context_builder_load_options{table_path_}; + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("commit_user_1"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + write_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -1201,10 +1240,15 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); - CommitContextBuilder commit_context_builder(table_path_, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path_}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); // Read selected MAP keys together with nested STRUCT sub-fields. @@ -1309,10 +1353,15 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1464,10 +1513,15 @@ TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path_}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); auto create_reader_result = table_read->CreateReader(data_splits); ASSERT_NOK_WITH_MSG(create_reader_result, "partial projection inside list"); } diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index 88240489f..5c4c7e8b9 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -29,6 +29,7 @@ #include "arrow/api.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/variant/generic_variant.h" #include "paimon/data/blob_descriptor.h" #include "paimon/defs.h" @@ -71,20 +72,30 @@ Result> ReadTable( std::string table_path = GetDataDir() + "/" + file_format + "/" + kDatabaseName + ".db/" + table_name; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); return ReadResultCollector::CollectResult(std::move(batch_reader)); diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index 153a7926c..390f7317b 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -31,6 +31,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" @@ -351,15 +352,19 @@ class PkCompactionInteTest : public ::testing::Test { int32_t bucket, const std::shared_ptr& write_array, int64_t commit_identifier, bool is_streaming = true, const std::vector& row_kinds = {}, bool write_only = true) const { - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(is_streaming) .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")); if (write_only) { - write_builder.AddOption(Options::WRITE_ONLY, "true"); + write_builder_options[Options::WRITE_ONLY] = "true"; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique(partition, bucket, row_kinds, &c_array); @@ -372,11 +377,15 @@ class PkCompactionInteTest : public ::testing::Test { Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -397,13 +406,18 @@ class PkCompactionInteTest : public ::testing::Test { const std::string& table_path, const std::map& partition, int32_t bucket, bool full_compaction, int64_t commit_identifier, const std::map& options = {}) { - WriteContextBuilder write_builder(table_path, "commit_user_1"); - write_builder.SetOptions(options); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); + write_builder_options = options; + write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "tmp")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); PAIMON_RETURN_NOT_OK(file_store_write->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( std::vector> commit_messages, @@ -422,16 +436,21 @@ class PkCompactionInteTest : public ::testing::Test { std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(table_path, options, /*is_streaming_mode=*/false)); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( - Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); if (predicate) { scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); std::vector> data_splits = result_plan->Splits(); @@ -456,15 +475,20 @@ class PkCompactionInteTest : public ::testing::Test { auto iter = expected_data_per_partition_bucket.find(key); ASSERT_TRUE(iter != expected_data_per_partition_bucket.end()) << "Unexpected partition=" << key.first << " bucket=" << key.second; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + if (predicate) { read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, @@ -490,20 +514,30 @@ class PkCompactionInteTest : public ::testing::Test { template void ScanAllRows(const std::string& table_path, Fn consume) { std::map options = {{Options::FILE_SYSTEM, "local"}}; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( - Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -617,13 +651,16 @@ TEST_F(PkCompactionInteTest, TestMetadataOnlyLevelUpgradeKeepsValueStats) { ASSERT_FALSE(before->value_stats == SimpleStats::EmptyStats()); ASSERT_EQ(before->value_stats, after->value_stats); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false) - .AddOption(Options::FILE_SYSTEM, "local") - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_SYSTEM] = "local"; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1u, plan->Splits().size()); auto split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -997,9 +1034,13 @@ TEST_F(PkCompactionInteTest, DeduplicateWithBitmap64HighPositions) { ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)); // Reload committed metadata and DV bytes; no test-created DV or lookup result is injected. - ScanContextBuilder scan_builder(table_path); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); int32_t dv_count = 0; for (const auto& split : plan->Splits()) { @@ -1746,10 +1787,15 @@ TEST_F(PkCompactionInteTest, CompactWithSchemaEvolution) { // Step 5: Read and verify using DataSplits from CompactAfter in write CommitMessages. { - ReadContextBuilder context_builder(table_path); - context_builder.AddOption(Options::FILE_FORMAT, "parquet"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "parquet"; + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); // Use the latest snapshot id for both splits. // The original table has 6 snapshots. Each write with force-compact produces @@ -2011,12 +2057,16 @@ TEST_F(PkCompactionInteTest, ContinuousWriteWithBackgroundCompact) { // Step 1-4: Streaming writes with background compact in one FileStoreWrite. { - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "tmp")); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); auto write_batch = [&](const std::string& json_data) -> Status { auto array = @@ -2220,12 +2270,15 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { auto data_type = arrow::struct_(fields); // Step 2: Write data to the branch-rt and compact. - WriteContextBuilder write_builder(table_path, "commit_user_1"); - write_builder.WithStreamingMode(true) - .WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")) - .WithBranch("rt"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); + write_builder_options["branch"] = "rt"; + write_builder.WithStreamingMode(true).WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); // dt=20240726, bucket-0 only has ["20240726", "cherry", 3] one row auto write_array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ @@ -2276,11 +2329,17 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(compact_after[0]->row_count, 2); // Step 4: Commit to branch-rt. - CommitContextBuilder commit_builder(table_path, "commit_user_1"); - commit_builder.AddOption(Options::FILE_SYSTEM, "local").AddOption(Options::BRANCH, "rt"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); + commit_builder_options[Options::FILE_SYSTEM] = "local"; + commit_builder_options[Options::BRANCH] = "rt"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); auto fs = std::make_shared(); @@ -2299,13 +2358,16 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { // Step 5: Scan and read branch-rt. std::map branch_options = {{Options::FILE_SYSTEM, "local"}, {Options::BRANCH, "rt"}}; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.WithStreamingMode(false) - .SetOptions(branch_options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = branch_options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); std::vector> splits = plan->Splits(); @@ -2316,11 +2378,15 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(split_impl->SnapshotId(), 3); } - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(branch_options); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = branch_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp index bdf7825a4..88f9fb24b 100644 --- a/test/inte/primary_key_sorted_index_inte_test.cpp +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -31,6 +31,7 @@ #include "arrow/api.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/table/source/data_split_impl.h" @@ -199,24 +200,27 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::optional& snapshot_id, bool index_enabled, const std::vector>& partition_filters, const std::string& branch, const std::shared_ptr& file_system) const { - ScanContextBuilder builder(TablePath(partitioned)); - builder.SetPredicate(predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, - index_enabled ? "true" : "false"); + TableLoadOptions builder_load_options{TablePath(partitioned)}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::GLOBAL_INDEX_ENABLED] = index_enabled ? "true" : "false"; + builder.SetPredicate(predicate); if (snapshot_id != std::nullopt) { - builder.AddOption(Options::SCAN_SNAPSHOT_ID, fmt::format("{}", snapshot_id.value())); + builder_options[Options::SCAN_SNAPSHOT_ID] = fmt::format("{}", snapshot_id.value()); } if (!partition_filters.empty()) { builder.SetPartitionFilter(partition_filters); } if (!branch.empty()) { - builder.AddOption(Options::BRANCH, branch); + builder_options[Options::BRANCH] = branch; } if (file_system != nullptr) { - builder.WithFileSystem(file_system); + builder_load_options.file_system = file_system; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); return table_scan->CreatePlan(); } @@ -224,7 +228,9 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::vector>& splits, const std::map& options, const std::shared_ptr& file_system) const { - ReadContextBuilder builder(TablePath(partitioned)); + TableLoadOptions builder_load_options{TablePath(partitioned)}; + std::map builder_options; + ReadContextBuilder builder; if (partitioned) { builder.SetReadFieldNames({"pt", "id", "score", "tag"}); } else { @@ -232,14 +238,15 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, } builder.SetPredicate(predicate).EnablePredicateFilter(true); for (const auto& [key, value] : options) { - builder.AddOption(key, value); + builder_options[key] = value; } if (file_system != nullptr) { - builder.WithFileSystem(file_system); + builder_load_options.file_system = file_system; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -541,13 +548,16 @@ TEST_P(PrimaryKeySortedIndexInteTest, ScoredSplitFailsBeforeForceKeepDeleteFallb inner_split, std::vector{Range(first_range.from, first_range.from)}, std::vector{0.5F}); - ReadContextBuilder builder(TablePath(/*partitioned=*/false)); + TableLoadOptions builder_load_options{TablePath(/*partitioned=*/false)}; + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"id", "score", "tag"}) .SetPredicate(predicate) .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); auto* key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read != nullptr); key_value_table_read->ForceKeepDelete(true); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 1988df48b..af1ef60e2 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -41,6 +41,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" @@ -283,22 +284,30 @@ Result> ReadSystemTable( const std::string& system_table_path, const std::map& options, bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, const std::vector& read_field_names = {}, bool read_next_plan = false) { - ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode); + TableLoadOptions scan_context_builder_load_options{system_table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder.WithStreamingMode(streaming_mode); if (predicate) { scan_context_builder.SetPredicate(predicate); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); if (read_next_plan) { PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan()); } - ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{system_table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + if (predicate) { read_context_builder.SetPredicate(predicate); } @@ -307,8 +316,10 @@ Result> ReadSystemTable( } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -380,18 +391,23 @@ TEST_P(ReadInteTest, TestAppendSimple) { DataField(2, arrow::field("f2", arrow::int32())), DataField(3, arrow::field("f3", arrow::float64()))}; - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true"); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.EnablePrefetch(param.enable_prefetch); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -463,18 +479,24 @@ TEST_P(ReadInteTest, TestReadWithLimits) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption(Options::READ_BATCH_SIZE, "1"); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options[Options::READ_BATCH_SIZE] = "1"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["orc.read.enable-metrics"] = "true"; context_builder.EnablePrefetch(param.enable_prefetch) .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .AddOption("orc.read.enable-metrics", "true") + .SetPrefetchBatchCount(10); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -526,14 +548,19 @@ TEST_P(ReadInteTest, TestReadAheadCacheMetrics) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") .SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -600,16 +627,21 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) { DataField(0, arrow::field("dt", arrow::utf8())), }; - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder.SetReadFieldNames({"dt"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector> data_splits; data_splits.reserve(3); @@ -659,17 +691,27 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ArrowSchemaRelease(&schema); std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); - ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + TableLoadOptions scan_context_builder_load_options{system_table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); - ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{system_table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); ASSERT_TRUE(result); @@ -691,17 +733,27 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; - ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + TableLoadOptions scan_context_builder_load_options{system_table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); - ReadContextBuilder read_context_builder(system_table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load_options{system_table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -938,17 +990,25 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTablesAtPinnedSnapshot) { const auto bound_schema = table->GetSchema(); auto create_reader = [&table](const std::map& read_options) -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan_context, - ScanContextBuilder("unused_path").SetOptions(read_options).Finish()); + std::map scan_context_builder_0_options; + ScanContextBuilder scan_context_builder_0; + scan_context_builder_0_options = read_options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_context_builder_0.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_1, + table->Copy(scan_context_builder_0_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - table->NewScan(std::move(scan_context))); + configured_table_1->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr read_context, - ReadContextBuilder("unused_path").SetOptions(read_options).Finish()); + std::map read_context_builder_48503_options; + ReadContextBuilder read_context_builder_48503; + read_context_builder_48503_options = read_options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_context_builder_48503.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_2, + table->Copy(read_context_builder_48503_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - table->NewRead(std::move(read_context))); + configured_table_2->NewRead(std::move(read_context))); return read->CreateReader(plan->Splits()); }; @@ -1109,23 +1169,32 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::shared_ptr predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); - ScanContextBuilder ro_scan_context_builder(table_path + "$ro"); - ro_scan_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions ro_scan_context_builder_load_options{table_path + "$ro"}; + std::map ro_scan_context_builder_options; + ScanContextBuilder ro_scan_context_builder; + ro_scan_context_builder_options = options; + ro_scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_scan_context, ro_scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_table_scan, - TableScan::Create(std::move(ro_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ro_table_scan, + TableScan::Create(ro_scan_context_builder_load_options, std::move(ro_scan_context), + ro_scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); ASSERT_EQ(ro_file_count, 1); - ReadContextBuilder read_context_builder(table_path + "$ro"); + TableLoadOptions read_context_builder_load_options{table_path + "$ro"}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; // Do not enable row-level predicate filtering: the result must come from the file reader's // row-group/page predicate pushdown. - read_context_builder.SetOptions(options).SetPredicate(predicate).SetReadFieldNames({"k", "v"}); + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate).SetReadFieldNames({"k", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(ro_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1169,11 +1238,15 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); - ScanContextBuilder scan_context_builder(table_path + "$ro"); - scan_context_builder.SetOptions(options); + TableLoadOptions scan_context_builder_load_options{table_path + "$ro"}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); auto projected_schema = arrow::schema({ @@ -1182,11 +1255,15 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { }); auto c_projected_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_projected_schema.get()).ok()); - ReadContextBuilder read_context_builder(table_path + "$ro"); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_projected_schema)); + TableLoadOptions read_context_builder_load_options{table_path + "$ro"}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1417,12 +1494,16 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite /*is_streaming_mode=*/true)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - WriteContextBuilder write_context_builder(table_path, "partial-write"); + TableLoadOptions write_context_builder_load_options{table_path}; + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("partial-write"); write_context_builder.WithWriteSchema({"score"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write, + FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + write_context_builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr batch, TestHelper::MakeRecordBatch(arrow::struct_({fields[2]}), R"([[10], [20]])", @@ -1432,11 +1513,15 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite write->PrepareCommit()); ASSERT_OK(write->Close()); - CommitContextBuilder commit_context_builder(table_path, "partial-write"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("partial-write"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_messages)); ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); @@ -1802,17 +1887,39 @@ TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadO for (bool direct : {false, true}) { SCOPED_TRACE(direct); const std::string path = table->GetRootPath() + "$" + kind; + TableLoadOptions scan_context_builder_48980_load_options{path}; + std::map scan_context_builder_48980_options; + ScanContextBuilder scan_context_builder_48980; + scan_context_builder_48980_options = read_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path).SetOptions(read_options).Finish()); + scan_context_builder_48980.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - direct ? table->NewScan(std::move(scan_context)) - : TableScan::Create(std::move(scan_context))); + direct ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->Copy(scan_context_builder_48980_options)); + return configured->NewScan(std::move(scan_context)); + })() + : TableScan::Create(scan_context_builder_48980_load_options, + std::move(scan_context), + scan_context_builder_48980_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + TableLoadOptions read_context_builder_93839_load_options{path}; + std::map read_context_builder_93839_options; + ReadContextBuilder read_context_builder_93839; + read_context_builder_93839_options = read_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(path).SetOptions(read_options).Finish()); + read_context_builder_93839.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - direct ? table->NewRead(std::move(read_context)) - : TableRead::Create(std::move(read_context))); + direct ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->Copy(read_context_builder_93839_options)); + return configured->NewRead(std::move(read_context)); + })() + : TableRead::Create(read_context_builder_93839_load_options, + std::move(read_context), + read_context_builder_93839_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1826,7 +1933,7 @@ TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadO } } -TEST(SystemTableReadInteTest, TestLoadedChangelogHonorsDynamicSequenceNumber) { +TEST(SystemTableReadInteTest, TestCopiedChangelogHonorsDynamicSequenceNumber) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()), @@ -1851,20 +1958,30 @@ TEST(SystemTableReadInteTest, TestLoadedChangelogHonorsDynamicSequenceNumber) { SCOPED_TRACE(kind); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("foo", "bar$" + kind))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured, table->Copy(read_options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_schema, + configured->GetSchema()->GetArrowSchema()); + auto copied = arrow::ImportSchema(copied_schema.get()); + ASSERT_TRUE(copied.ok()) << copied.status().ToString(); + ASSERT_EQ(4, copied.ValueOrDie()->num_fields()); for (bool direct : {false, true}) { SCOPED_TRACE(direct); const std::string path = table->GetRootPath() + "$" + kind; + TableLoadOptions scan_load{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - ScanContextBuilder(path).SetOptions(read_options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - direct ? table->NewScan(std::move(scan_context)) - : TableScan::Create(std::move(scan_context))); + ScanContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + direct ? configured->NewScan(std::move(scan_context)) + : TableScan::Create(scan_load, std::move(scan_context), read_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + TableLoadOptions read_load{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, - ReadContextBuilder(path).SetOptions(read_options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - direct ? table->NewRead(std::move(read_context)) - : TableRead::Create(std::move(read_context))); + ReadContextBuilder().Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + direct ? configured->NewRead(std::move(read_context)) + : TableRead::Create(read_load, std::move(read_context), read_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -2171,17 +2288,23 @@ TEST_P(ReadInteTest, TestAppendReadWithMultipleBuckets) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder + .EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2252,19 +2375,22 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnablePrefetch(param.enable_prefetch) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .AddOption("orc.read.enable-metrics", "true"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnablePrefetch( + param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2354,18 +2480,23 @@ TEST_P(ReadInteTest, TestAppendReadWithComplexTypePredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_data.db/append_complex_data"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f6", "f2", "f4", "f3", "f5"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2428,19 +2559,25 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder + .SetPredicate(predicate) .EnableLateMaterializing(false) .EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2512,18 +2649,24 @@ TEST_P(ReadInteTest, TestAppendReadWithLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder + .SetPredicate(predicate) .EnableLateMaterializing(true) .EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2590,19 +2733,25 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f3", "f0", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder + .SetPredicate(predicate) .EnablePredicateFilter(true) .EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2676,18 +2825,22 @@ TEST_P(ReadInteTest, TestAppendReadIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - ReadContextBuilder context_builder(paimon::test::GetDataDir() + "/" + param.file_format + - "/append_09.db/append_09/"); + TableLoadOptions context_builder_load_options; + context_builder_load_options.path = + paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09/"; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = TableRead::Create( + context_builder_load_options, std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(data_splits); @@ -2722,16 +2875,21 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVectorSimple) { data_file = "data-ed5d7184-cf42-4f4a-bf83-1e3080d9012d-0.parquet"; dv_file = "index/index-62372b7b-d1cf-4dde-a162-c9d9d35006b4-1"; } - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); DataSplitsDv input_data_splits = { {path + "f1=10/bucket-1", @@ -2771,16 +2929,20 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVector) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2") - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; std::string deletion_file; @@ -2838,19 +3000,24 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot6) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(15.0)); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -2924,17 +3091,22 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot8) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f0", "f3", "f1"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3003,20 +3175,25 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolution) { DataField(5, arrow::field("a", arrow::int32())), DataField(8, arrow::field("e", arrow::int32()))}; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3110,15 +3287,20 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithReaderBuildParallelism /*schema ids*/ {1, 0}}}; auto data_splits = CreateDataSplits(input_data_splits, /*snapshot_id=*/2); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3175,20 +3357,25 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateFilter) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/append_table_with_alter_table/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3255,18 +3442,23 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateOnlyPushDown) std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/" "append_table_with_alter_table/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3329,17 +3521,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot5WithSchemaEvolution) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3415,17 +3612,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolution) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3501,18 +3703,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .EnableLateMaterializing(false) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3587,18 +3793,22 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithLateMaterializing FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetPredicate(predicate); - context_builder.EnableLateMaterializing(true) - .EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnableLateMaterializing(true).EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3671,19 +3881,24 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter) FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); context_builder.EnablePredicateFilter(true); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3764,17 +3979,22 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithBuildInFieldId) { "data-11b12094-192f-4ad8-92a8-ae8cba5e25ef-0.parquet"}; } - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"key0", "key1", "k", "c", "d", "a", "e"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); DataSplitsSchema input_data_splits = { {path + "key0=0/key1=1/bucket-0", BinaryRowGenerator::GenerateRow({0, 1}, pool_.get()), @@ -3828,16 +4048,21 @@ TEST_P(ReadInteTest, TestAppendReadNestedType) { DataField(7, arrow::field("f6", arrow::decimal128(2, 2)))}; std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -3884,17 +4109,22 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCast) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -3967,18 +4197,23 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCastWithPredicatePushD std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("read.batch-size", "2"); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["read.batch-size"] = "2"; + context_builder.SetPredicate(predicate); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list_0; std::vector file_list_1; @@ -4047,17 +4282,21 @@ TEST_P(ReadInteTest, TestReadWithPKFallBackBranch) { DataField(2, arrow::field("amount", arrow::int32())), }; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption(Options::FILE_FORMAT, param.file_format) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4103,11 +4342,15 @@ TEST_P(ReadInteTest, TestReadWithAppendFallBackBranch) { data_splits.push_back(split); } - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.EnablePrefetch(param.enable_prefetch); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4147,13 +4390,17 @@ TEST_P(ReadInteTest, TestFallBackBranchStreamRead) { std::vector read_fields = {DataField(0, arrow::field("dt", arrow::utf8())), DataField(1, arrow::field("name", arrow::utf8())), DataField(2, arrow::field("amount", arrow::int32()))}; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_split)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4193,17 +4440,21 @@ TEST_P(ReadInteTest, TestReadWithPKRtBranch) { DataField(2, arrow::field("amount", arrow::int32())), }; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .WithBranch("rt"); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["branch"] = "rt"; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4250,17 +4501,21 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) { DataField(2, arrow::field("value2", arrow::int32())), }; - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", - param.enable_adaptive_prefetch_strategy) - .WithBranch("test"); + context_builder_options["test.enable-adaptive-prefetch-strategy"] = + param.enable_adaptive_prefetch_strategy; + context_builder_options["branch"] = "test"; + context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4381,15 +4636,20 @@ TEST_P(ReadInteTest, TestSpecificFs) { auto countable_fs = std::make_shared(std::make_shared(), &io_count); - ReadContextBuilder context_builder(path); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); - context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.EnablePrefetch(param.enable_prefetch) - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true") - .WithFileSystem(countable_fs); + context_builder_options[Options::FILE_FORMAT] = param.file_format; + + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder_load_options.file_system = countable_fs; + context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); std::vector file_list; if (param.file_format == "orc") { @@ -4449,20 +4709,28 @@ Result> ReadGlobalSystemTable( std::string sys_path = PathUtil::JoinPath(PathUtil::JoinPath(warehouse, "sys"), table_name); - ScanContextBuilder scan_context_builder(sys_path); - scan_context_builder.SetOptions(options); + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_7, + system_table->Copy(scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(std::move(scan_context))); + configured_table_7->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(sys_path); - read_context_builder.SetOptions(options); + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_8, + system_table->Copy(read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(std::move(read_context))); + configured_table_8->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 7aceaa6a7..640c03487 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -45,6 +45,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/predicate/equal.h" @@ -544,13 +545,16 @@ class RealtimeWriteInteTest : public ::testing::Test { Result> CreateRealtimeWriter( const std::shared_ptr& realtime_context, const std::string& temp_directory) const { - WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options_; + builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); if (!temp_directory.empty()) { builder.WithTempDirectory(temp_directory); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(std::move(context)); + return FileStoreWrite::Create(builder_load_options, std::move(context), builder_options); } Result> CreateRealtimeWriter() const { @@ -724,11 +728,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result Commit(const std::vector& realtime_commits, int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, commit_user_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); return commit->CommitWithProgress(realtime_commits, commit_identifier, /*watermark=*/std::nullopt); } @@ -749,11 +756,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result DropPartition(const std::map& partition, int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, kDropPartitionCommitUser); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(kDropPartitionCommitUser); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->DropPartition({partition}, commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -766,11 +776,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result TruncateTable(int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, kTruncateCommitUser); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(kTruncateCommitUser); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->TruncateTable(commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -783,11 +796,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result RollbackToAsLatest(int64_t target_snapshot_id) const { - CommitContextBuilder builder(table_path_, kRollbackCommitUser); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(kRollbackCommitUser); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(bool rolled_back, commit->RollbackToAsLatest(target_snapshot_id)); if (!rolled_back) { return Status::Invalid("failed to commit rollback snapshot"); @@ -805,11 +821,15 @@ class RealtimeWriteInteTest : public ::testing::Test { Result>> CompactAndPrepareCommit( const std::vector& partition_buckets, int64_t commit_identifier, bool full_compaction, std::unique_ptr* compactor) const { - WriteContextBuilder builder(table_path_, "regression-compactor"); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder("regression-compactor"); + builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).WithStreamingMode(true).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - FileStoreWrite::Create(std::move(context))); + builder.WithStreamingMode(true).Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(context), builder_options)); for (const RealtimePartitionBucket& scope : partition_buckets) { PAIMON_RETURN_NOT_OK(writer->Compact(scope.partition, scope.bucket, full_compaction)); } @@ -821,11 +841,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CommitCompaction(const std::vector>& messages, int64_t commit_identifier) const { - CommitContextBuilder builder(table_path_, "regression-compactor"); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("regression-compactor"); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->Commit(messages, commit_identifier)); SnapshotManager snapshot_manager(dir_->GetFileSystem(), table_path_); PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, snapshot_manager.LatestSnapshot()); @@ -838,12 +861,17 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CompactAndCommit(const std::map& partition, int32_t bucket, int64_t commit_identifier, bool full_compaction = true) const { - WriteContextBuilder write_builder(table_path_, commit_user_); - write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory( + TableLoadOptions write_builder_load_options{table_path_}; + std::map write_builder_options; + WriteContextBuilder write_builder(commit_user_); + write_builder_options = options_; + write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "compact-tmp")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr compaction_writer, - FileStoreWrite::Create(std::move(write_context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr compaction_writer, + FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options)); PAIMON_RETURN_NOT_OK(compaction_writer->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( std::vector> compaction_messages, @@ -852,11 +880,16 @@ class RealtimeWriteInteTest : public ::testing::Test { return Status::Invalid("compaction did not produce a commit message"); } - CommitContextBuilder commit_builder(table_path_, commit_user_); + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder(commit_user_); + commit_builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr compaction_commit, - FileStoreCommit::Create(std::move(commit_context))); + commit_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr compaction_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); PAIMON_RETURN_NOT_OK(compaction_commit->Commit(compaction_messages, commit_identifier)); PAIMON_RETURN_NOT_OK(compaction_writer->Close()); @@ -881,40 +914,49 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result ExpireSnapshots() const { - CommitContextBuilder builder(table_path_, commit_user_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); return commit->Expire(); } Result> CreatePlan( const std::shared_ptr& realtime_context, const std::shared_ptr& predicate) const { - ScanContextBuilder scan_builder(table_path_); + TableLoadOptions scan_builder_load_options{table_path_}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; if (realtime_context) { scan_builder.WithRealtimeContext(realtime_context); } scan_builder.SetPredicate(predicate).WithMemoryPool(pool_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, - scan_builder.SetOptions(options_).Finish()); + scan_builder_options = options_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); return scan->CreatePlan(); } Result> CreateQueryReader( const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); return table_read->CreateReader(plan->Splits()); } @@ -929,16 +971,19 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::shared_ptr& plan, const std::shared_ptr& realtime_context, const std::vector& read_fields, const std::shared_ptr& predicate, bool enable_predicate_filter) const { - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames(read_fields) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames(read_fields) .SetPredicate(predicate) .EnablePredicateFilter(enable_predicate_filter) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -952,14 +997,17 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::string& expected_json) const { std::unique_ptr c_read_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok()); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadSchema(std::move(c_read_schema)) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadSchema(std::move(c_read_schema)) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1045,13 +1093,15 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CountRows(const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .WithRealtimeContext(realtime_context) - .WithMemoryPool(pool_); + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr count_reader, table_read->CreateCountReader(plan->Splits())); return count_reader->CountRows(); @@ -1344,31 +1394,45 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - WriteContextBuilder write_builder(table_path_, commit_user_); - write_builder.SetOptions(disabled_options) - .WithStreamingMode(true) - .WithRealtimeContext(realtime_context); + TableLoadOptions write_builder_load_options{table_path_}; + std::map write_builder_options; + WriteContextBuilder write_builder(commit_user_); + write_builder_options = disabled_options; + write_builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + write_builder_options), "real-time write requires realtime.enabled=true"); - ScanContextBuilder scan_builder(table_path_); - scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); + TableLoadOptions scan_builder_load_options{table_path_}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = disabled_options; + scan_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), - "real-time scan requires realtime.enabled=true"); - - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); + ASSERT_NOK_WITH_MSG( + TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options), + "real-time scan requires realtime.enabled=true"); + + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = disabled_options; + read_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), - "real-time read requires realtime.enabled=true"); - - CommitContextBuilder commit_builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(disabled_options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + ASSERT_NOK_WITH_MSG( + TableRead::Create(read_builder_load_options, std::move(read_context), read_builder_options), + "real-time read requires realtime.enabled=true"); + + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder(commit_user_); + commit_builder_options = disabled_options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -1468,14 +1532,16 @@ TEST_F(RealtimeWriteInteTest, TestRefreshPreparationAllowsScanWriteAndStoreRegis ConfigureRealtimeDeduplicateTable(); auto fs = std::make_shared(dir_->GetFileSystem()); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_) - .WithStreamingMode(true) - .WithRealtimeContext(context) - .WithFileSystem(fs); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options_; + builder_load_options.file_system = fs; + builder.WithStreamingMode(true).WithRealtimeContext(context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); const std::vector initial = {{1, "old", "p0"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(initial, /*partitioned=*/false)); @@ -1579,14 +1645,16 @@ TEST_F(RealtimeWriteInteTest, TestRefreshPreparationFailureKeepsProgressAndCanRe ConfigureRealtimeDeduplicateTable(); auto fs = std::make_shared(dir_->GetFileSystem()); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); - WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_) - .WithStreamingMode(true) - .WithRealtimeContext(context) - .WithFileSystem(fs); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + WriteContextBuilder builder(commit_user_); + builder_options = options_; + builder_load_options.file_system = fs; + builder.WithStreamingMode(true).WithRealtimeContext(context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); const std::vector initial = {{1, "old", "p0"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(initial, /*partitioned=*/false)); @@ -1837,11 +1905,16 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) auto concurrent_message = std::make_shared( initial_message->Partition(), /*bucket=*/0, /*total_buckets=*/1, DataIncrement({}, {}, {}, {concurrent_dv}, {}), CompactIncrement({}, {}, {}, {}, {})); - CommitContextBuilder concurrent_builder(table_path_, "concurrent-dv-writer"); + TableLoadOptions concurrent_builder_load_options{table_path_}; + std::map concurrent_builder_options; + CommitContextBuilder concurrent_builder("concurrent-dv-writer"); + concurrent_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_context, - concurrent_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_commit, - FileStoreCommit::Create(std::move(concurrent_context))); + concurrent_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr concurrent_commit, + FileStoreCommit::Create(concurrent_builder_load_options, std::move(concurrent_context), + concurrent_builder_options)); bool interleaved = false; auto fs = std::make_shared( PathUtil::JoinPath(table_path_, "snapshot/snapshot-2"), [&]() -> Status { @@ -1849,11 +1922,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) return concurrent_commit->Commit({concurrent_message}, /*commit_identifier=*/0, /*watermark=*/std::nullopt); }); - CommitContextBuilder builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/1, /*watermark=*/std::nullopt)); @@ -1915,11 +1992,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryRemapsCompactedO interleaved = true; return Status::OK(); }); - CommitContextBuilder builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/2, /*watermark=*/std::nullopt)); @@ -1966,11 +2047,15 @@ TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompa ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot)); } - WriteContextBuilder write_builder(table_path_, "compactor"); + TableLoadOptions write_builder_load_options{table_path_}; + std::map write_builder_options; + WriteContextBuilder write_builder("compactor"); + write_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.SetOptions(options_).WithStreamingMode(true).Finish()); + write_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr compactor, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr forbidden_batch, MakeBatch({{99, "not-written", "p0"}}, /*partitioned=*/false)); ASSERT_NOK_WITH_MSG(compactor->Write(std::move(forbidden_batch)), "only supports compaction"); @@ -2043,11 +2128,16 @@ TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompa } else { ASSERT_OK(publish_build()); } - CommitContextBuilder commit_builder(table_path_, "compactor"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).WithFileSystem(fs).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("compactor"); + commit_builder_options = options_; + commit_builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK(commit->Commit(messages, /*commit_identifier=*/0)); ASSERT_TRUE(interleaved); // Only unpublished intermediate outputs and their sidecars are reclaimed after publication. @@ -2173,11 +2263,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCompactionRejectsReplacedIn ASSERT_EQ(1, initial_snapshot); ASSERT_OK(writer->Close()); - WriteContextBuilder write_builder(table_path_, "stale-compactor"); + TableLoadOptions write_builder_load_options{table_path_}; + std::map write_builder_options; + WriteContextBuilder write_builder("stale-compactor"); + write_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.SetOptions(options_).WithStreamingMode(true).Finish()); + write_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr compactor, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_OK(compactor->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( std::vector> messages, @@ -2190,11 +2284,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCompactionRejectsReplacedIn ASSERT_OK_AND_ASSIGN(Snapshot replacement, CompactAndCommit(/*partition=*/{}, /*bucket=*/0, /*commit_identifier=*/1)); ASSERT_EQ(2, replacement.Id()); - CommitContextBuilder commit_builder(table_path_, "stale-compactor"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("stale-compactor"); + commit_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->Commit(messages, /*commit_identifier=*/0), "input is no longer active"); ASSERT_OK_AND_ASSIGN(std::vector rows, ReadRows()); @@ -2736,11 +2834,14 @@ TEST_F(RealtimeWriteInteTest, TestNoHitPreservesBaselineDv) { auto no_hit = std::make_shared(); no_hit->Add(0); progress[0].deleted_offsets = no_hit; - CommitContextBuilder builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder(commit_user_); + builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot, commit->CommitWithProgress(progress, /*commit_identifier=*/4, /*watermark=*/std::nullopt)); @@ -2864,11 +2965,15 @@ TEST_F(RealtimeWriteInteTest, TestPublicationConflictCleansOnlyAttemptDv) { } return Status::OK(); }); - CommitContextBuilder builder(table_path_, "regression-compactor"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).WithFileSystem(fs).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("regression-compactor"); + builder_options = options_; + builder_load_options.file_system = fs; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK(commit->Commit(messages, /*commit_identifier=*/0)); ASSERT_OK_AND_ASSIGN(uint64_t attempts, commit->GetCommitMetrics()->GetCounter( CommitMetrics::LAST_COMMIT_ATTEMPTS)); @@ -3801,18 +3906,26 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesSnapshotConflict) { const std::string first_snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"); auto conflict_file_system = std::make_shared(first_snapshot_path); - CommitContextBuilder bucket0_builder(table_path_, "bucket0_realtime_commit_user"); + TableLoadOptions bucket0_builder_load_options{table_path_}; + std::map bucket0_builder_options; + CommitContextBuilder bucket0_builder("bucket0_realtime_commit_user"); + bucket0_builder_options = options_; + bucket0_builder_load_options.file_system = conflict_file_system; + ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket0_context, bucket0_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr bucket0_context, - bucket0_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket0_commit, - FileStoreCommit::Create(std::move(bucket0_context))); - CommitContextBuilder bucket1_builder(table_path_, "bucket1_realtime_commit_user"); + std::unique_ptr bucket0_commit, + FileStoreCommit::Create(bucket0_builder_load_options, std::move(bucket0_context), + bucket0_builder_options)); + TableLoadOptions bucket1_builder_load_options{table_path_}; + std::map bucket1_builder_options; + CommitContextBuilder bucket1_builder("bucket1_realtime_commit_user"); + bucket1_builder_options = options_; + bucket1_builder_load_options.file_system = conflict_file_system; + ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket1_context, bucket1_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr bucket1_context, - bucket1_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket1_commit, - FileStoreCommit::Create(std::move(bucket1_context))); + std::unique_ptr bucket1_commit, + FileStoreCommit::Create(bucket1_builder_load_options, std::move(bucket1_context), + bucket1_builder_options)); std::optional> bucket0_result; std::optional> bucket1_result; @@ -3880,13 +3993,18 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id)); } - WriteContextBuilder compact_write_builder(table_path_, "compaction_write_user"); - compact_write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory( + TableLoadOptions compact_write_builder_load_options{table_path_}; + std::map compact_write_builder_options; + WriteContextBuilder compact_write_builder("compaction_write_user"); + compact_write_builder_options = options_; + compact_write_builder.WithStreamingMode(true).WithTempDirectory( PathUtil::JoinPath(dir_->Str(), "conflict-compact-tmp")); ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_write_context, compact_write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_writer, - FileStoreWrite::Create(std::move(compact_write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compact_writer, + FileStoreWrite::Create(compact_write_builder_load_options, std::move(compact_write_context), + compact_write_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( @@ -3910,18 +4028,28 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { PathUtil::JoinPath(table_path_, "snapshot/snapshot-3"); auto conflict_file_system = std::make_shared(conflict_snapshot_path); - CommitContextBuilder append_commit_builder(table_path_, commit_user_); + TableLoadOptions append_commit_builder_load_options{table_path_}; + std::map append_commit_builder_options; + CommitContextBuilder append_commit_builder(commit_user_); + append_commit_builder_options = options_; + append_commit_builder_load_options.file_system = conflict_file_system; + ASSERT_OK_AND_ASSIGN(std::unique_ptr append_commit_context, + append_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr append_commit_context, - append_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr append_commit, - FileStoreCommit::Create(std::move(append_commit_context))); - CommitContextBuilder compact_commit_builder(table_path_, "compaction_commit_user"); + std::unique_ptr append_commit, + FileStoreCommit::Create(append_commit_builder_load_options, + std::move(append_commit_context), append_commit_builder_options)); + TableLoadOptions compact_commit_builder_load_options{table_path_}; + std::map compact_commit_builder_options; + CommitContextBuilder compact_commit_builder("compaction_commit_user"); + compact_commit_builder_options = options_; + compact_commit_builder_load_options.file_system = conflict_file_system; + ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_commit_context, + compact_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( - std::unique_ptr compact_commit_context, - compact_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_commit, - FileStoreCommit::Create(std::move(compact_commit_context))); + std::unique_ptr compact_commit, + FileStoreCommit::Create(compact_commit_builder_load_options, + std::move(compact_commit_context), compact_commit_builder_options)); std::optional> append_result; std::optional compact_status; @@ -4185,14 +4313,17 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) { ASSERT_OK_AND_ASSIGN(std::shared_ptr lifetime_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(lifetime_plan->Splits())); ASSERT_FALSE(query_view->expired()); @@ -4260,10 +4391,14 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::vector disk_rows, ReadRows()); ASSERT_EQ(rows, disk_rows); - ScanContextBuilder scan_builder(table_path_ + "$ro"); - scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_); + TableLoadOptions scan_builder_load_options{table_path_ + "$ro"}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options = options_; + scan_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - Result> scan = TableScan::Create(std::move(scan_context)); + Result> scan = + TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options); ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString(); ASSERT_NE(std::string::npos, scan.status().ToString().find( "PK real-time union read does not support read-optimized")); @@ -4293,14 +4428,17 @@ TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) { CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_OK_AND_ASSIGN(std::shared_ptr reader_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_reader, table_read->CreateReader(reader_plan->Splits())); @@ -4795,14 +4933,17 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) { }); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadSchema(std::move(c_schema)) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadSchema(std::move(c_schema)) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -5090,22 +5231,31 @@ TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) { TEST_F(RealtimeWriteInteTest, TestPkRecovery) { CreatePkTable(); - WriteContextBuilder seed_builder(table_path_, commit_user_); - seed_builder.SetOptions(options_).WithStreamingMode(true); + TableLoadOptions seed_builder_load_options{table_path_}; + std::map seed_builder_options; + WriteContextBuilder seed_builder(commit_user_); + seed_builder_options = options_; + seed_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_context, seed_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_writer, - FileStoreWrite::Create(std::move(seed_context))); + FileStoreWrite::Create(seed_builder_load_options, std::move(seed_context), + seed_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_batch, MakeLogicalBatch({Row{99, "seed", "p0"}}, /*bucket=*/0)); ASSERT_OK(seed_writer->Write(std::move(seed_batch))); ASSERT_OK_AND_ASSIGN(std::vector> seed_messages, seed_writer->PrepareCommit(/*wait_compaction=*/false, /*commit_identifier=*/0)); - CommitContextBuilder seed_commit_builder(table_path_, commit_user_); + TableLoadOptions seed_commit_builder_load_options{table_path_}; + std::map seed_commit_builder_options; + CommitContextBuilder seed_commit_builder(commit_user_); + seed_commit_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_commit_context, - seed_commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_commit, - FileStoreCommit::Create(std::move(seed_commit_context))); + seed_commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr seed_commit, + FileStoreCommit::Create(seed_commit_builder_load_options, std::move(seed_commit_context), + seed_commit_builder_options)); ASSERT_OK(seed_commit->Commit(seed_messages, /*commit_identifier=*/0)); ASSERT_OK(seed_writer->Close()); const std::vector mutations = { @@ -5192,11 +5342,16 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context)); ASSERT_EQ(0, memory_usage); } - WriteContextBuilder compact_builder(table_path_, commit_user_); - compact_builder.SetOptions(options_).WithStreamingMode(true); + TableLoadOptions compact_builder_load_options{table_path_}; + std::map compact_builder_options; + WriteContextBuilder compact_builder(commit_user_); + compact_builder_options = options_; + compact_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_context, compact_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_writer, - FileStoreWrite::Create(std::move(compact_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr compact_writer, + FileStoreWrite::Create(compact_builder_load_options, std::move(compact_context), + compact_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( @@ -5224,11 +5379,15 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { std::max(compacted_live_max_sequence_number, file->max_sequence_number); } ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber); - CommitContextBuilder commit_builder(table_path_, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + TableLoadOptions commit_builder_load_options{table_path_}; + std::map commit_builder_options; + CommitContextBuilder commit_builder(commit_user_); + commit_builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); ASSERT_OK(commit->Commit(compact_messages, /*commit_identifier=*/4)); ASSERT_OK(compact_writer->Close()); @@ -5548,11 +5707,14 @@ TEST_F(RealtimeWriteInteTest, TestCommitWithProgressRejectsCoveredRangesFromAnot writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); ASSERT_OK(Commit(commits, /*commit_identifier=*/0)); - CommitContextBuilder builder(table_path_, "another_realtime_commit_user"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr context, - builder.SetOptions(options_).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(context))); + TableLoadOptions builder_load_options{table_path_}; + std::map builder_options; + CommitContextBuilder builder("another_realtime_commit_user"); + builder_options = options_; + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit, + FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(commits, /*commit_identifier=*/0, /*watermark=*/std::nullopt), "another commit user or identifier"); @@ -6368,14 +6530,17 @@ TEST_F(RealtimeWriteInteTest, TestSuccessfulReaderCreationConsumesRealtimeSplitT ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr first_reader, table_read->CreateReader(plan->Splits())); first_reader->Close(); @@ -6400,16 +6565,19 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_EQ(1, plan->Splits().size()); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .SetPredicate(std::make_shared()) .EnablePredicateFilter(true) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()), "cannot cast predicate unsupported"); @@ -7751,14 +7919,17 @@ TEST_F(RealtimeWriteInteTest, TestReaderPinsMemoryAcrossRefresh) { std::dynamic_pointer_cast(plan->Splits()[0]); ASSERT_NE(nullptr, realtime_split); - ReadContextBuilder read_builder(table_path_); - read_builder.SetOptions(options_) - .SetReadFieldNames({"id", "payload", "pt"}) + TableLoadOptions read_builder_load_options{table_path_}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options = options_; + read_builder.SetReadFieldNames({"id", "payload", "pt"}) .WithRealtimeContext(realtime_context) .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context_impl, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 17807e254..019aa567f 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -36,6 +36,7 @@ #include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -144,21 +145,23 @@ class ScanAndReadInteTest : public testing::Test, } } - void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder) { - read_context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); + void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder, + std::map& read_options) { + read_options["test.enable-adaptive-prefetch-strategy"] = "false"; if (EnablePrefetch()) { read_context_builder->EnablePrefetch(true).SetPrefetchBatchCount(3); } } - Result> FinishScanContext(ScanContextBuilder& builder) { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& builder_options) { if (EnableSnapshotLiveManifestCache()) { if (!snapshot_live_manifest_cache_) { snapshot_live_manifest_cache_ = std::make_shared(/*max_weight=*/64 * 1024 * 1024); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(snapshot_live_manifest_cache_); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(snapshot_live_manifest_cache_); } return builder.Finish(); } @@ -240,12 +243,17 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - Result> scan_context = FinishScanContext(scan_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + Result> scan_context = + FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(std::move(scan_context).value()); + TableScan::Create(scan_context_builder_load_options, std::move(scan_context).value(), + scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); CHECK_HOOK_STATUS(result_plan.status(), i); @@ -254,12 +262,16 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(splits); @@ -297,12 +309,17 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - Result> scan_context = FinishScanContext(scan_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + Result> scan_context = + FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(std::move(scan_context).value()); + TableScan::Create(scan_context_builder_load_options, std::move(scan_context).value(), + scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); CHECK_HOOK_STATUS(result_plan.status(), i); @@ -310,12 +327,16 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(splits); @@ -350,21 +371,31 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -392,10 +423,16 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -403,10 +440,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -505,21 +546,22 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { bool entry_cache) -> Result { std::shared_ptr pool = GetMemoryPool(); scan_pools.push_back(pool); - ScanContextBuilder builder(table_path); - builder.WithMemoryPool(pool) - .WithExecutor(executor) - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, entry_cache ? "3" : "0") - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - lazy_decode ? "true" : "false") - .AddOption(Options::READ_BATCH_SIZE, "128"); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = entry_cache ? "3" : "0"; + builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + lazy_decode ? "true" : "false"; + builder_options[Options::READ_BATCH_SIZE] = "128"; + builder.WithMemoryPool(pool).WithExecutor(executor).WithCache(cache); if (entry_cache) { // No explicit bucket or partition filter: the predicate must select the bucket. builder.SetPredicate(predicate); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); PAIMON_ASSIGN_OR_RAISE(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); @@ -537,11 +579,14 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { (historical && data_split->Bucket() == kBuckets / 2 - 1)); ASSERT_EQ(data_split->GetFileList().size(), 1); } - ReadContextBuilder builder(table_path); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ReadContextBuilder builder; builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(result.plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -690,15 +735,17 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) } auto cache = std::make_shared(capacities); for (int32_t attempt = 0; attempt < 2; ++attempt) { - ScanContextBuilder builder(table_path); - builder.SetPredicate(predicate) - .WithCache(cache) - .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED, - lazy_decode ? "true" : "false"); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder_options[Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED] = + lazy_decode ? "true" : "false"; + builder.SetPredicate(predicate).WithCache(cache); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -707,12 +754,16 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) ASSERT_OK_AND_ASSIGN(uint64_t hit, scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); ASSERT_EQ(hit, attempt); - ReadContextBuilder read_builder(table_path); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; read_builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -779,10 +830,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { // All buckets have overlapping stats for this key. An explicit filter bypasses // inference, proving that ordinary statistics cannot account for the pruning. for (int32_t bucket = 0; bucket < kNumBuckets; ++bucket) { - ScanContextBuilder builder(table_path); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; builder.SetPredicate(predicate).SetBucketFilter(bucket); - ASSERT_OK_AND_ASSIGN(auto context, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto context, FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(builder_load_options, + std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); @@ -804,23 +858,33 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); - ScanContextBuilder scan_builder(table_path); - scan_builder.SetPredicate(lookup_predicate) - .AddOption(Options::BUCKET, current_bucket_count); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::BUCKET] = current_bucket_count; + scan_builder.SetPredicate(lookup_predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_builder, scan_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto scan, TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( uint64_t enabled, scan->GetMetrics()->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); - ScanContextBuilder cached_builder(table_path); - cached_builder.SetPredicate(lookup_predicate) - .AddOption(Options::BUCKET, current_bucket_count); - ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, - TableScan::Create(std::move(cached_context))); + TableLoadOptions cached_builder_load_options{table_path}; + std::map cached_builder_options; + ScanContextBuilder cached_builder; + cached_builder_options[Options::BUCKET] = current_bucket_count; + cached_builder.SetPredicate(lookup_predicate); + ASSERT_OK_AND_ASSIGN(auto cached_context, + FinishScanContext(cached_builder, cached_builder_options)); + ASSERT_OK_AND_ASSIGN( + auto cached_scan, + TableScan::Create(cached_builder_load_options, std::move(cached_context), + cached_builder_options)); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -833,11 +897,15 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { ASSERT_TRUE(data_split); ASSERT_EQ(data_split->Bucket(), bucket_ids[lookup_index]); } - ReadContextBuilder read_builder(table_path); - AddReadOptionsForPrefetch(&read_builder); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; + AddReadOptionsForPrefetch(&read_builder, read_builder_options); read_builder.SetPredicate(lookup_predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + auto read, TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(reader))); @@ -861,10 +929,16 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -872,10 +946,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -911,18 +989,28 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5WithReaderBuildParallelism) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -953,15 +1041,25 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 1, 2, 3, 4}; auto expected_snapshot1 = std::make_shared( @@ -1002,10 +1100,16 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_10.db/append_10"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1013,10 +1117,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1041,10 +1149,16 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { "/append_complex_build_in_fieldid.db/" "append_complex_build_in_fieldid/"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1069,10 +1183,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { std::shared_ptr arrow_data_type = DataField::ConvertDataFieldsToArrowStructType(read_fields); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1096,14 +1214,24 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // normal batch scan case for pk+dv, all data in level 0 is filtered out - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1137,15 +1265,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1160,16 +1298,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1196,8 +1344,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // predicate: f0 != "Alice" (key predicate) and f3 > 18 (value predicate) and all data in level // 0 - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1207,14 +1357,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1237,8 +1394,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1248,14 +1407,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1278,15 +1444,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder.SetBucketFilter(0).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will be applied, as in kv mode value filter is enabled auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -1300,15 +1472,24 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { // normal batch scan case for pk+dv, all data in level 0 is filtered out, set row limits to 6, // data in partition 20 is truncated - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -1336,17 +1517,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from an compact snapshot std::vector> expected_snapshot_ids = {4, 5, 6}; @@ -1388,17 +1578,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, second plan is snapshot6 // (isStreaming=true) @@ -1434,15 +1633,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot6 with merge, level 0 data in base manifest // list is contained @@ -1471,17 +1680,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from the first snapshot std::vector> expected_snapshot_ids = {1, 3, 5, 6}; @@ -1530,17 +1748,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // use from-snapshot mode, first plan is empty, second plan is snapshot 3 with delta files // (snapshot 2 is compact snapshot, so skipped) @@ -1575,18 +1802,27 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_nested_type.db/pk_table_nested_type/"; - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN( auto read_context, read_context_builder.SetReadFieldNames({"shopId", "dt", "hr", "col0", "col1", "col2"}) .Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -1629,14 +1865,23 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // normal batch scan case for pk+mor, use latest snapshot if not specified - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1676,15 +1921,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { // normal batch scan case for pk+mor, read snapshot 2 (append snapshot), all data is in level 0 // with merge read - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -1720,16 +1975,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // all data not in partition 10, bucket 1 is filtered out - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1758,8 +2023,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; // predicate: f0 != "Alice" and f0 < "Lucy" and f3 <= 30.0 // as f3 <= 30.0 is a value filter, it does not take effect @@ -1780,14 +2047,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1821,8 +2095,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; std::string literal_str = "Alice"; auto not_equal = PredicateBuilder::NotEqual( @@ -1837,14 +2113,21 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -1869,16 +2152,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will be applied as all data files in bucket 1 in compact snapshot 3 are // all filtered auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1890,20 +2179,31 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1930,20 +2230,31 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "rbm32"); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -1957,24 +2268,35 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "sum"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + scan_context_builder_options["fields.f3.aggregate-function"] = "sum"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will not be applied, as merge engine is aggregation auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; + read_context_builder_options["fields.f3.aggregate-function"] = "sum"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "aggregation") - .AddOption("fields.f3.aggregate-function", "sum"); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -2001,24 +2323,35 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::MERGE_ENGINE, "partial-update") - .AddOption(Options::IGNORE_DELETE, "true"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; + scan_context_builder_options[Options::IGNORE_DELETE] = "true"; + scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); // predicate f3 > 20.0 will not be applied, as merge engine is partial-update auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + read_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; + read_context_builder_options[Options::IGNORE_DELETE] = "true"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.AddOption(Options::MERGE_ENGINE, "partial-update") - .AddOption(Options::IGNORE_DELETE, "true"); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -2047,15 +2380,24 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { // in pk mor mode, limit does not take effect, as we do not know the number of records after // merging - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -2087,17 +2429,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // first plan is snapshot4 (isStreaming=false), second plan is snapshot5 std::vector> expected_snapshot_ids = {4, 5}; @@ -2132,17 +2483,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {1, 2, 4, 5}; auto expected_snapshot1_batch = std::make_shared( @@ -2193,17 +2553,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { // use from-snapshot mode, first plan is empty, snapshot 2 in second plan only includes delta // files - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 2, 4, 5}; auto expected_snapshot2_stream = std::make_shared( @@ -2238,21 +2607,30 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(50.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // first plan is snapshot 5 with empty plan, second plan is snapshot 5 with delta files // (predicates take no effects in delta mode) @@ -2274,24 +2652,30 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamPredicateFilter) { std::string table_path = paimon::test::GetDataDir() + FileFormat() + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") - .AddOption(Options::SCAN_MODE, "from-snapshot") - .WithStreamingMode(true); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot"; + scan_context_builder.WithStreamingMode(true); std::shared_ptr predicate = PredicateBuilder::GreaterThan( /*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(31.05)); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(scan_context_builder)); + FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 5}; auto expected_snapshot5_stream = std::make_shared( @@ -2313,15 +2697,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; // normal batch scan case for pk+first row, all data in level 0 is filtered out - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -2353,17 +2747,26 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from a compact snapshot std::vector> expected_snapshot_ids = {3, 4, 5}; @@ -2406,15 +2809,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, all level 0 data is merged std::vector> expected_snapshot_ids = {5}; @@ -2445,15 +2858,24 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_09.db/pk_09/"; // normal batch scan case for pk+dv (09 version) - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 8); @@ -2482,29 +2904,37 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { auto file_format = FileFormat(); - auto check_result = - [&](const std::string& table_path, - const std::vector>& partition_filters, - const std::shared_ptr& expected) { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - - ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ASSERT_EQ(result_plan->SnapshotId().value(), 1); - ASSERT_OK_AND_ASSIGN(auto batch_reader, - table_read->CreateReader(result_plan->Splits())); - ASSERT_OK_AND_ASSIGN(auto read_result, - ReadResultCollector::CollectResult(std::move(batch_reader))); - // check result - ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); - }; + auto check_result = [&](const std::string& table_path, + const std::vector>& + partition_filters, + const std::shared_ptr& expected) { + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder.SetPartitionFilter(partition_filters); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); + + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 1); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, + ReadResultCollector::CollectResult(std::move(batch_reader))); + // check result + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + }; auto expected_without_partition_filter = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([ @@ -2552,28 +2982,36 @@ TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { "/append_with_empty_partition_with_empty_value.db/" "append_with_empty_partition_with_empty_value/"; - auto check_result = - [&](const std::vector>& partition_filters, - const std::shared_ptr& expected) { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - - ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ASSERT_EQ(result_plan->SnapshotId().value(), 1); - ASSERT_OK_AND_ASSIGN(auto batch_reader, - table_read->CreateReader(result_plan->Splits())); - ASSERT_OK_AND_ASSIGN(auto read_result, - ReadResultCollector::CollectResult(std::move(batch_reader))); - // check result - ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); - }; + auto check_result = [&](const std::vector>& + partition_filters, + const std::shared_ptr& expected) { + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder.SetPartitionFilter(partition_filters); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); + + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 1); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, + ReadResultCollector::CollectResult(std::move(batch_reader))); + // check result + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + }; auto expected_without_partition_filter = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([ @@ -2604,10 +3042,16 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_09.db/append_09/"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2616,12 +3060,16 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { // read std::shared_ptr read_pool = GetMemoryPool(); { - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.WithMemoryPool(read_pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2705,26 +3153,41 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { { // batch scan - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; scan_context_builder.WithStreamingMode(false); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_TRUE(result_plan->Splits().empty()); } { // stream scan: from snapshot 1 - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 1, 2}; auto expected_snapshot1 = std::make_shared( @@ -2749,16 +3212,26 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot 2 - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 2}; auto expected_snapshot2 = std::make_shared( @@ -2775,17 +3248,26 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot full - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .AddOption(Options::SCAN_MODE, "from-snapshot-full") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + scan_context_builder_options[Options::SCAN_MODE] = "from-snapshot-full"; + scan_context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); std::vector> expected_snapshot_ids = {1, 2}; auto expected_snapshot1 = std::make_shared( @@ -2817,22 +3299,31 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar } std::string table_path = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table"; - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; auto predicate = PredicateBuilder::LessThan( /*field_index=*/3, /*field_name=*/"f4", FieldType::INT, Literal(300006)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_EQ(result_plan->Splits().size(), 1); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"f10", "f8", "f4", "f13"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2868,11 +3359,17 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { auto predicate = PredicateBuilder::GreaterThan( /*field_index=*/2, /*field_name=*/"key_2", FieldType::INT, Literal(500)); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); ASSERT_EQ(result_plan->Splits().size(), 1); @@ -2885,11 +3382,15 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { } ASSERT_EQ(data_file_count, 1); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2921,20 +3422,30 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { paimon::test::GetDataDir() + "/append_table_with_multiple_file_format.db/append_table_with_multiple_file_format"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -2960,19 +3471,28 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_no_external.db/pk_dv_index_in_data_no_external"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3000,19 +3520,28 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_no_external.db/pk_dv_index_not_in_data_no_external"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3042,9 +3571,14 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_with_external.db/external"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3053,10 +3587,14 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/false, &splits); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3087,22 +3625,30 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { Literal(FieldType::STRING, "Alice0", 6)); // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN( - auto scan_context, - scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options["file-index.read.enabled"] = "false"; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption("file-index.read.enabled", "false"); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options["file-index.read.enabled"] = "false"; + + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3162,24 +3708,33 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(15)); - ScanContextBuilder stats_only_scan_builder(table_path); - stats_only_scan_builder.SetOptions(options) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false") - .SetPredicate(absent_predicate); + TableLoadOptions stats_only_scan_builder_load_options{table_path}; + std::map stats_only_scan_builder_options; + ScanContextBuilder stats_only_scan_builder; + stats_only_scan_builder_options = options; + stats_only_scan_builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + stats_only_scan_builder.SetPredicate(absent_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stats_only_scan, + TableScan::Create(stats_only_scan_builder_load_options, std::move(stats_only_scan_context), + stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // The embedded bitmap knows that 15 is absent and eliminates the file during scan planning. - ScanContextBuilder indexed_scan_builder(table_path); - indexed_scan_builder.SetOptions(options).SetPredicate(absent_predicate); + TableLoadOptions indexed_scan_builder_load_options{table_path}; + std::map indexed_scan_builder_options; + ScanContextBuilder indexed_scan_builder; + indexed_scan_builder_options = options; + indexed_scan_builder.SetPredicate(absent_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan_context, indexed_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan, - TableScan::Create(std::move(indexed_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr indexed_scan, + TableScan::Create(indexed_scan_builder_load_options, std::move(indexed_scan_context), + indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); @@ -3188,22 +3743,29 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // the other rows. const std::shared_ptr present_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(20)); - ScanContextBuilder retained_scan_builder(table_path); - retained_scan_builder.SetOptions(options).SetPredicate(present_predicate); + TableLoadOptions retained_scan_builder_load_options{table_path}; + std::map retained_scan_builder_options; + ScanContextBuilder retained_scan_builder; + retained_scan_builder_options = options; + retained_scan_builder.SetPredicate(present_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr retained_scan_context, retained_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr retained_scan, - TableScan::Create(std::move(retained_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr retained_scan, + TableScan::Create(retained_scan_builder_load_options, std::move(retained_scan_context), + retained_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr retained_plan, retained_scan->CreatePlan()); ASSERT_FALSE(retained_plan->Splits().empty()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options) - .SetPredicate(present_predicate) - .EnablePredicateFilter(false); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(present_predicate).EnablePredicateFilter(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(retained_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3263,24 +3825,33 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); // The evolved min/max range is still [10, 30], so statistics alone retain the old file. - ScanContextBuilder stats_only_scan_builder(table_path); - stats_only_scan_builder.SetOptions(options) - .AddOption(Options::FILE_INDEX_READ_ENABLED, "false") - .SetPredicate(predicate); + TableLoadOptions stats_only_scan_builder_load_options{table_path}; + std::map stats_only_scan_builder_options; + ScanContextBuilder stats_only_scan_builder; + stats_only_scan_builder_options = options; + stats_only_scan_builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; + stats_only_scan_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr stats_only_scan, + TableScan::Create(stats_only_scan_builder_load_options, std::move(stats_only_scan_context), + stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // Reconstructing the predicate to schema-0 lets its embedded bitmap prove that 15 is absent. - ScanContextBuilder indexed_scan_builder(table_path); - indexed_scan_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions indexed_scan_builder_load_options{table_path}; + std::map indexed_scan_builder_options; + ScanContextBuilder indexed_scan_builder; + indexed_scan_builder_options = options; + indexed_scan_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan_context, indexed_scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr indexed_scan, - TableScan::Create(std::move(indexed_scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr indexed_scan, + TableScan::Create(indexed_scan_builder_load_options, std::move(indexed_scan_context), + indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); } @@ -3293,9 +3864,14 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_with_external.db/external"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3304,10 +3880,14 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/true, &splits); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3334,9 +3914,14 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_multiple_ts_precision_and_timezone.db" "/append_with_multiple_ts_precision_and_timezone/"; - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3344,10 +3929,14 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3382,9 +3971,14 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { "/append_with_cast_timestamp.db" "/append_with_cast_timestamp/"; // scan - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3392,10 +3986,14 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3430,18 +4028,24 @@ TEST_F(ScanAndReadInteTest, TestParquetDictionaryBinaryStoredArrowSchema) { "/parquet/append_dictionary_binary.db/" "append_dictionary_binary"; - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->SnapshotId().value()); ASSERT_EQ(1, plan->Splits().size()); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3510,8 +4114,12 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { const std::vector& projection, const std::shared_ptr& expected_result, bool exact_filter = false) { - ScanContextBuilder scan_builder(table_path); - ReadContextBuilder read_builder(table_path); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; if (predicate) { scan_builder.SetPredicate(predicate); if (projection.empty()) { @@ -3523,14 +4131,19 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { read_builder.SetReadFieldNames(projection); } read_builder.EnablePredicateFilter(exact_filter); - read_builder.AddOption("read.batch-size", "1"); + read_builder_options["read.batch-size"] = "1"; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(2)); ASSERT_FALSE(plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3601,21 +4214,31 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { auto check = [&](int64_t snapshot_id, const std::vector& projection, const std::shared_ptr& expected_result) { SCOPED_TRACE("snapshot " + std::to_string(snapshot_id)); - ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(snapshot_id)); ASSERT_FALSE(plan->Splits().empty()); - ReadContextBuilder read_builder(table_path); - read_builder.AddOption("read.batch-size", "1"); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options["read.batch-size"] = "1"; + if (!projection.empty()) { read_builder.SetReadFieldNames(projection); } ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3675,21 +4298,31 @@ TEST_F(ScanAndReadInteTest, TestLanceLtzCompatibility) { const std::vector& projection, const std::shared_ptr& expected_result) { SCOPED_TRACE("snapshot " + std::to_string(snapshot_id)); - ScanContextBuilder scan_builder(table_path); - scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_builder_load_options{table_path}; + std::map scan_builder_options; + ScanContextBuilder scan_builder; + scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, + TableScan::Create(scan_builder_load_options, std::move(scan_context), + scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(snapshot_id)); ASSERT_FALSE(plan->Splits().empty()); - ReadContextBuilder read_builder(table_path); - read_builder.AddOption("read.batch-size", "1"); + TableLoadOptions read_builder_load_options{table_path}; + std::map read_builder_options; + ReadContextBuilder read_builder; + read_builder_options["read.batch-size"] = "1"; + if (!projection.empty()) { read_builder.SetReadFieldNames(projection); } ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto read, + TableRead::Create(read_builder_load_options, std::move(read_context), + read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); ASSERT_TRUE(actual); @@ -3757,22 +4390,30 @@ TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) { auto check_compatibility = [](const std::string& table_path, const std::shared_ptr& predicate, const std::shared_ptr& expected_result) { - ScanContextBuilder scan_context_builder(table_path); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; if (predicate != nullptr) { scan_context_builder.SetPredicate(predicate); read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3821,15 +4462,21 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibility) { const std::string table_name = "append_java_compat"; const std::string table_path = GetDataDir() + "/vortex/" + table_name + ".db/" + table_name; - ScanContextBuilder scan_context_builder(table_path); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3849,17 +4496,25 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityLtzAcrossTimezones) { const std::string timezones[] = {"UTC", "Asia/Shanghai"}; for (const std::string& tz : timezones) { TimezoneGuard timezone_guard(tz); - ScanContextBuilder scan_context_builder(table_path); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load_options, std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3924,15 +4579,21 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityAllTypes) { const std::string table_name = "all_types_java_compat"; const std::string table_path = GetDataDir() + "/vortex/" + table_name + ".db/" + table_name; - ScanContextBuilder scan_context_builder(table_path); - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -3947,22 +4608,32 @@ TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) { auto read_data = [](int64_t snapshot_id, const std::string& result_json) { std::string table_path = GetDataDir() + "/avro/append_multiple.db/append_multiple"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), snapshot_id); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false"); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + read_context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4022,10 +4693,15 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { std::string table_path = GetDataDir() + "/avro/pk_with_multiple_type.db/pk_with_multiple_type"; // scan - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), snapshot_id); @@ -4033,10 +4709,14 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { ASSERT_EQ(1, splits.size()); // read - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4095,32 +4775,47 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { // Historical files use two buckets even when the current option has changed. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + + scan_context_builder_options[Options::BUCKET] = current_bucket_count; + scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); - ScanContextBuilder cached_builder(table_path); - cached_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6") - .AddOption(Options::BUCKET, current_bucket_count) - .SetPartitionFilter({{{"f1", "10"}}}) - .SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); + TableLoadOptions cached_builder_load_options{table_path}; + std::map cached_builder_options; + ScanContextBuilder cached_builder; + cached_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; + cached_builder_options[Options::BUCKET] = current_bucket_count; + cached_builder.SetPartitionFilter({{{"f1", "10"}}}).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto cached_context, + FinishScanContext(cached_builder, cached_builder_options)); + ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(cached_builder_load_options, + std::move(cached_context), + cached_builder_options)); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -4168,18 +4863,27 @@ TEST_P(ScanAndReadInteTest, TestReadNullableMapKey) { const std::string table_path = paimon::test::GetDataDir() + "orc/nullable_map_key.db/nullable_map_key"; - ScanContextBuilder scan_context_builder(table_path); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->SnapshotId()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_EQ(result_plan->Splits().size(), 1); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::READ_BATCH_SIZE, "1"); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::READ_BATCH_SIZE] = "1"; + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); const std::vector expected_rows = {R"([[0, 1, [["one", 10]]]])", @@ -4211,10 +4915,14 @@ TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); // CreateCountReader with empty splits should return 0 rows. std::vector> empty_splits; @@ -4229,25 +4937,39 @@ TEST_P(ScanAndReadInteTest, TestCountRowsConsistencyWithCreateReader) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Scan latest snapshot - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Method 1: CreateCountReader + iterate batches - ReadContextBuilder count_context_builder(table_path); - AddReadOptionsForPrefetch(&count_context_builder); + TableLoadOptions count_context_builder_load_options{table_path}; + std::map count_context_builder_options; + ReadContextBuilder count_context_builder; + AddReadOptionsForPrefetch(&count_context_builder, count_context_builder_options); ASSERT_OK_AND_ASSIGN(auto count_read_context, count_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto count_table_read, TableRead::Create(std::move(count_read_context))); + ASSERT_OK_AND_ASSIGN( + auto count_table_read, + TableRead::Create(count_context_builder_load_options, std::move(count_read_context), + count_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto count_reader, count_table_read->CreateCountReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(int64_t count_result, count_reader->CountRows()); // Method 2: CreateReader + iterate batches - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -4263,19 +4985,28 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithPredicateNotSupported) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Set predicate in read context. CountReader currently does not support predicate pushdown. auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(result_plan->Splits()), "predicate pushdown is not supported"); @@ -4287,15 +5018,24 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithForceKeepDeleteNotSupported "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. - ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(scan_context_builder, scan_context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ReadContextBuilder read_context_builder(table_path); - AddReadOptionsForPrefetch(&read_context_builder); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); auto* key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read != nullptr); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3f949484d..1a2a02200 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -32,6 +32,7 @@ #include "arrow/api.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/decimal_utils.h" @@ -64,15 +65,16 @@ enum class ManifestCacheMode { NoCache, Cache }; class ScanInteTest : public testing::TestWithParam { public: - Result> FinishScanContext(ScanContextBuilder& builder) { + Result> FinishScanContext( + ScanContextBuilder& builder, std::map& builder_options) { if (GetParam() == ManifestCacheMode::Cache) { if (!cache_) { cache_ = std::make_shared(std::map{ {CacheKind::MANIFEST, 64 * 1024 * 1024}, {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); } - builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") - .WithCache(cache_); + builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; + builder.WithCache(cache_); } return builder.Finish(); } @@ -269,10 +271,15 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; auto run_scan = [&]() -> Result> { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithCache(manifest_cache); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.WithCache(manifest_cache); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, + TableScan::Create(context_builder_load_options, + std::move(scan_context), context_builder_options)); return table_scan->CreatePlan(); }; @@ -293,19 +300,23 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { Result>> RunBucketSnapshotScan( const std::string& table_path, int64_t snapshot_id, int32_t bucket, const std::shared_ptr& cache, int32_t max_snapshot_live_manifest_versions) { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) - .SetBucketFilter(bucket); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + context_builder.SetBucketFilter(bucket); if (max_snapshot_live_manifest_versions > 0) { - context_builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, - std::to_string(max_snapshot_live_manifest_versions)); + context_builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = + std::to_string(max_snapshot_live_manifest_versions); } if (cache) { context_builder.WithCache(cache); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); std::vector> data_splits; @@ -396,11 +407,16 @@ TEST(ScanInteManifestCacheTest, TestSnapshotLiveManifestCacheRetainsSnapshotsPer TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -453,10 +469,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -514,13 +536,19 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { std::string table_path = data_dir + "orc/append_09.db/append_09"; auto check_result = [&](const std::optional& specific_table_schema) { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + if (specific_table_schema) { - context_builder.SetTableSchema(specific_table_schema.value()); + context_builder_load_options.table_schema_json = specific_table_schema.value(); } - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, + std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -578,10 +606,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { TEST_P(ScanInteTest, TestScanInvalidSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "100"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "100"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "The specified scan snapshotId 100 is out of available snapshotId range [1, 5]."); @@ -589,10 +623,16 @@ TEST_P(ScanInteTest, TestScanInvalidSnapshot) { TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // batch scan multiple ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "end of scan"); @@ -600,13 +640,19 @@ TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") - .AddOption(Options::SOURCE_SPLIT_OPEN_FILE_COST, "1024") - .AddOption(Options::SOURCE_SPLIT_TARGET_SIZE, "2048"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder_options[Options::SOURCE_SPLIT_OPEN_FILE_COST] = "1024"; + context_builder_options[Options::SOURCE_SPLIT_TARGET_SIZE] = "2048"; + // open cost = 1024, and split target size is 2048, indicates at most 2 files in a split - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -670,10 +716,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3").SetLimit(3); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder.SetLimit(3); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -716,10 +768,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.SetBucketFilter(0).AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; + context_builder.SetBucketFilter(0); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -759,10 +817,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { // from snapshot is specified std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder1_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -871,11 +935,16 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -925,10 +994,16 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -981,11 +1056,16 @@ TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); std::vector> expected_snapshot_ids = {}; CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, {}); @@ -994,12 +1074,17 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) - .AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder2_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1066,13 +1151,18 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshotFull().ToString()) - .AddOption(Options::SCAN_SNAPSHOT_ID, "2") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshotFull().ToString(); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); ASSERT_TRUE(scan_context); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder1_1( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1151,20 +1241,30 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, + std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::BUCKET] = "-2"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options), "do not support bucket=-2 in scan process"); } } @@ -1177,10 +1277,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, val.data(), val.size())); - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1223,12 +1329,16 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { PredicateBuilder::And({greater_than_predicate, less_than_predicate, not_in_predicate, not_equal})); - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate) - .AddOption(Options::SCAN_SNAPSHOT_ID, "1") - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate).WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); DataSplitImpl::Builder builder1_2( BinaryRowGenerator::GenerateRow({10}, pool_.get()), @@ -1284,10 +1394,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { std::map partition_keys; partition_keys["f1"] = "10"; - ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1330,10 +1446,15 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithInvalidPartitionFilter) { std::map partition_keys; partition_keys["invalid_partition_key"] = "10"; - ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options), "field invalid_partition_key does not exist in partition keys"); } @@ -1347,12 +1468,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF std::map partition_keys; partition_keys["f1"] = "10"; - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate) - .SetPartitionFilter({partition_keys}) - .AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate).SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1399,10 +1524,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { partition_keys["f1"] = "10"; partition_keys["f2"] = "0"; - ScanContextBuilder context_builder(table_path); - context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPartitionFilter({partition_keys}); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1455,10 +1586,16 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) Literal(paimon::Decimal(23, 5, 0))); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan); // check snapshot id @@ -1516,10 +1653,16 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) Literal(FieldType::BINARY, "zoo", 3)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); - ScanContextBuilder context_builder(table_path); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -1571,12 +1714,18 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1668,16 +1817,22 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); auto equal = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Emily", 5)); auto predicate = PredicateBuilder::And({greater_than, equal}).value(); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1714,10 +1869,16 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").SetBucketFilter(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetBucketFilter(2); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_TRUE(result_plan->Splits().empty()); @@ -1726,19 +1887,29 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::BUCKET] = "-1"; + context_builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options), "do not support pk table bucket=-1 in scan process"); } TEST_P(ScanInteTest, TestReadWithNoSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_nested_type.db/append_table_with_nested_type"; - ScanContextBuilder context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->SnapshotId()); ASSERT_TRUE(result_plan->Splits().empty()); @@ -1748,7 +1919,9 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_alter_table_with_cast.db/append_table_alter_table_with_cast"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::Or( @@ -1770,9 +1943,13 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { FieldType::INT, Literal(80)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2, child3})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1813,7 +1990,9 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table.db/append_table_with_alter_table"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::GreaterThan(/*field_index=*/2, /*field_name=*/"k", FieldType::INT, Literal(36)); @@ -1821,9 +2000,13 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { Literal(96)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1891,7 +2074,9 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table_with_dense_field.db/" "append_table_with_alter_table_with_dense_field"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::GreaterThan(/*field_index=*/2, /*field_name=*/"f1", FieldType::INT, Literal(22)); @@ -1899,9 +2084,13 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { FieldType::DOUBLE, Literal(0.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child3})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -1939,7 +2128,9 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Tony", 4)); @@ -1947,9 +2138,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -2016,7 +2211,9 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Lucy", 4)); @@ -2024,9 +2221,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } @@ -2035,7 +2236,9 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Lucy", 4)); @@ -2043,9 +2246,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { Literal(10)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits auto result_data_splits = CollectDataSplits(result_plan); @@ -2083,14 +2290,20 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; // file0 will be removed as f5 not exists auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT, Literal(100)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2152,15 +2365,21 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; auto child1 = PredicateBuilder::IsNull(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT); auto child2 = PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2227,7 +2446,9 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; // in stats filter: predicate is trimmed as type for f1 is not consist: int->bigint // in index filter: predicate is removed as type for f1 is converted bigint->int, which is not @@ -2235,9 +2456,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, Literal(100l)); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2304,7 +2529,9 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - ScanContextBuilder context_builder(table_path); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; // child1 will remove file1 for schema1 auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, @@ -2314,9 +2541,13 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { Literal(FieldType::STRING, "David", 5)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); - context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; + context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } @@ -2324,11 +2555,16 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { TEST_P(ScanInteTest, TestScanAppendWithTag1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_TAG_NAME, "1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_TAG_NAME] = "1"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, - FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(1, result_plan->SnapshotId().value()); @@ -2337,10 +2573,16 @@ TEST_P(ScanInteTest, TestScanAppendWithTag1) { TEST_P(ScanInteTest, TestScanInvalidTag) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; - ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::SCAN_TAG_NAME, "unknown"); - ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ScanContextBuilder context_builder; + context_builder_options[Options::SCAN_TAG_NAME] = "unknown"; + + ASSERT_OK_AND_ASSIGN(auto scan_context, + FinishScanContext(context_builder, context_builder_options)); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(context_builder_load_options, std::move(scan_context), + context_builder_options)); ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "Tag 'unknown' doesn't exist."); } @@ -2353,38 +2595,62 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { // EarlierOrEqual(snap3.time) → snap-3 { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 3); } // EarlierOrEqual(snap3.time - 1) → snap-2 { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis() - 1)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis() - 1); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 2); } // EarlierOrEqual(INT64_MAX) → snap-5 (latest) { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, - std::to_string(std::numeric_limits::max())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = + std::to_string(std::numeric_limits::max()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 5); } // EarlierOrEqual(0) → no snapshot found, expect error { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "There is currently no snapshot earlier than or equal to timestamp"); } @@ -2400,10 +2666,16 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { // T=0: no snapshot earlier than T=0, stream starts from snapshot 1. { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2411,11 +2683,16 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap2.time+1: EarlierThan(snap2.time+1)=snap2, stream starts from snapshot 3. { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap2.TimeMillis() + 1)) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap2.TimeMillis() + 1); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2423,11 +2700,16 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap3.time: EarlierThan(snap3.time)=snap2, stream starts from snapshot 3. { - ScanContextBuilder builder(table_path); - builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2435,13 +2717,17 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=INT64_MAX: EarlierThan(INT64_MAX)=snap5, stream starts from snapshot 6 (beyond latest). { - ScanContextBuilder builder(table_path); - builder - .AddOption(Options::SCAN_TIMESTAMP_MILLIS, - std::to_string(std::numeric_limits::max())) - .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; + builder_options[Options::SCAN_TIMESTAMP_MILLIS] = + std::to_string(std::numeric_limits::max()); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + FinishScanContext(builder, builder_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + TableScan::Create(builder_load_options, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_TRUE(plan0->Splits().empty()); diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index c211f4425..278c81a51 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -35,6 +35,7 @@ #include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/reader/reader_utils.h" @@ -170,13 +171,18 @@ class WriteAndReadInteTest const std::map& partition, const std::set& buckets, bool full_compaction, int64_t commit_identifier) const { - WriteContextBuilder write_context_builder(table_path, "commit_user"); + TableLoadOptions write_context_builder_load_options{table_path}; + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("commit_user"); + write_context_builder.WithTempDirectory(LookupTempDirectory()); + write_context_builder_options = options; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, + write_context_builder.WithStreamingMode(true).Finish()); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr write_context, - write_context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + std::unique_ptr file_store_write, + FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + write_context_builder_options)); for (int32_t bucket : buckets) { PAIMON_RETURN_NOT_OK(file_store_write->Compact(partition, bucket, full_compaction)); } @@ -187,11 +193,16 @@ class WriteAndReadInteTest if (compact_messages.empty()) { return Status::Invalid("expected compaction commit messages"); } - CommitContextBuilder commit_context_builder(table_path, "commit_user"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user"); + commit_context_builder_options = options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_store_commit, + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options)); return file_store_commit->Commit(compact_messages, commit_identifier); } @@ -202,10 +213,15 @@ class WriteAndReadInteTest std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(read_fields); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(read_fields); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -225,10 +241,15 @@ class WriteAndReadInteTest auto c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_schema.get())); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadSchema(std::move(c_schema)); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -257,11 +278,16 @@ class WriteAndReadInteTest Result> InnerScan( const std::map& options) const { std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options).AddOption(Options::SCAN_MODE, - StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); return table_scan->CreatePlan(); } @@ -459,16 +485,20 @@ TEST_P(WriteAndReadInteTest, TestAppendReadWithNestedPredicateAcrossBatches) { SCOPED_TRACE(batch_size); for (bool enable_filter : {false, true}) { SCOPED_TRACE(enable_filter); - ReadContextBuilder builder(table_path); - builder.SetOptions(options) - .SetReadFieldNames({"payload", "value", "id", "key"}) + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options = options; + builder_options[Options::READ_BATCH_SIZE] = std::to_string(batch_size); + builder.SetReadFieldNames({"payload", "value", "id", "key"}) .SetPredicate(predicate) .EnablePredicateFilter(enable_filter) .EnablePrefetch(false) - .EnableLateMaterializing(false) - .AddOption(Options::READ_BATCH_SIZE, std::to_string(batch_size)); + .EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + auto table_read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); ASSERT_TRUE(actual); @@ -634,19 +664,28 @@ TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(2)); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -716,21 +755,30 @@ TEST_P(WriteAndReadInteTest, TestAppendWithExternalBitmapAndRangeBitmapIndexes) ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({name_predicate, score_predicate})); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->Splits().size()); // Keep precise post-read filtering disabled. The exact result therefore verifies that the // bitmap and range-bitmap indexes produced by the write path are consumed by the read path. - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -828,13 +876,16 @@ TEST_P(WriteAndReadInteTest, TestLuminaSearchAfterRenameAcrossSchemaVersions) { auto search = std::make_shared("vec", 3, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(path); - builder.SetOptions(current_options) - .SetReadFieldNames({"id", "_vec_INDEX_SCORE"}) + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options = current_options; + builder.SetReadFieldNames({"id", "_vec_INDEX_SCORE"}) .SetVectorSearch(search) .EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + auto read, TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -926,14 +977,17 @@ TEST_P(WriteAndReadInteTest, TestLuminaVectorIndexAfterCompaction) { auto search = std::make_shared( "embedding", limit, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(path); - builder.SetOptions(options) - .SetReadFieldNames({"id", "_embedding_INDEX_SCORE"}) + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options = options; + builder.SetReadFieldNames({"id", "_embedding_INDEX_SCORE"}) .SetVectorSearch(search) .EnablePreciseFileIndexEvaluate(true); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr actual, @@ -1048,13 +1102,16 @@ TEST_P(WriteAndReadInteTest, TestAppendWithLuminaVectorIndex) { "embedding", /*limit=*/3, std::vector{0.0f, 0.0f}, /*pre_filter=*/nullptr, /*predicate=*/nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - ReadContextBuilder builder(PathUtil::JoinPath(root, "foo.db/bar")); + TableLoadOptions builder_load_options{PathUtil::JoinPath(root, "foo.db/bar")}; + std::map builder_options; + ReadContextBuilder builder; builder.SetReadFieldNames({"id", "_embedding_INDEX_SCORE"}) .SetVectorSearch(search) .EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -3311,11 +3368,15 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3324,10 +3385,14 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { // of the result is attributable to split/file/RG/page pruning, not to a // post-read row-level filter. This is what makes the exact assertion below // meaningful as a check that page-index filtering is wired and working. - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3421,11 +3486,15 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3433,10 +3502,14 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { // Read with predicate but WITHOUT EnablePredicateFilter, so the narrowing // observed below is attributable to page-index filtering rather than a // post-read row-level filter. - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3507,12 +3580,16 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { {Literal(7), Literal(11), Literal(15)}); ASSERT_TRUE(predicate); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3521,15 +3598,19 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { // therefore finds no data for it and skips ahead into the next row group it owns, whose first // selected row sits in the middle of that row group. Row level filtering stays off: the // expected rows below are exactly what page-index filtering selects. - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options) - .SetPredicate(predicate) + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + read_context_builder.SetPredicate(predicate) .EnablePrefetch(true) .SetPrefetchMaxParallelNum(2) - .SetPrefetchBatchCount(3) - .AddOption("test.enable-adaptive-prefetch-strategy", "false"); + .SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3596,10 +3677,15 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { auto cache = std::make_shared(CacheKind::DATA_FILE_FOOTER, 128 * 1024 * 1024); auto read_once = [&]() -> Result { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (result_plan->SnapshotId() != std::optional(1)) { return Status::Invalid("unexpected snapshot id"); @@ -3608,10 +3694,14 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { return Status::Invalid("no splits found"); } - ReadContextBuilder read_context_builder(table_path); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; read_context_builder.WithCache(cache); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(result_plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3918,10 +4008,16 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { (void)helper; std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - WriteContextBuilder write_context_builder(table_path, "commit_user_1"); + TableLoadOptions write_context_builder_load_options{table_path}; + std::map write_context_builder_options; + WriteContextBuilder write_context_builder("commit_user_1"); + write_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + write_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, @@ -3938,27 +4034,41 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - commit_context_builder.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + commit_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + std::move(commit_context), + commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(10)); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options) - .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) - .SetPredicate(predicate); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); + scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -5015,10 +5125,15 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionRea auto read_fields = [&](const std::vector& field_names) -> Status { PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames(field_names); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -6083,12 +6198,16 @@ TEST_P(LanceTimestampLtzInteTest, TestWriteAndReadInPinnedTimezone) { ASSERT_OK_AND_ASSIGN(std::vector> splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames( + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames( {"ltz_row", "ltz_added", "ltz_9", "id", "ltz_array", "ltz_0", "ltz_to_ntz", "ntz_to_ltz"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, @@ -6237,16 +6356,19 @@ TEST_P(TimeWriteAndReadInteTest, TestPKTableWithTimeColumns) { auto create_plan = [&](const std::shared_ptr& predicate) -> Result> { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options).AddOption(Options::SCAN_MODE, - StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); if (predicate) { scan_context_builder.SetPredicate(predicate); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + TableScan::Create(scan_context_builder_load, std::move(scan_context), + scan_context_builder_options)); return table_scan->CreatePlan(); }; auto planned_buckets = [](const Plan& plan) -> Result> { @@ -6264,15 +6386,18 @@ TEST_P(TimeWriteAndReadInteTest, TestPKTableWithTimeColumns) { const std::shared_ptr result_schema = arrow::schema(fields_with_row_kind); auto read = [&](const Plan& plan, const std::shared_ptr& predicate) -> Result> { - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options); + TableLoadOptions read_context_builder_load{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; if (predicate) { read_context_builder.SetPredicate(predicate).EnablePredicateFilter(true); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + TableRead::Create(read_context_builder_load, std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan.Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr actual, @@ -6469,16 +6594,20 @@ TEST_P(TimeWriteAndReadInteTest, TestAppendTableWithTimePartitionAndValuePredica .ValueOrDie()); for (bool enable_scan_predicate : {false, true}) { SCOPED_TRACE(enable_scan_predicate ? "with scan predicate" : "without scan predicate"); - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetOptions(options).AddOption( - Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + TableLoadOptions scan_context_builder_load{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options = options; + scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); if (enable_scan_predicate) { scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(scan_context_builder_load, std::move(scan_context), + scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); std::set planned_partitions; for (const auto& split : plan->Splits()) { @@ -6490,13 +6619,17 @@ TEST_P(TimeWriteAndReadInteTest, TestAppendTableWithTimePartitionAndValuePredica : std::set({kEarlyPart, kLatePart}), planned_partitions); - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetPredicate(predicate).EnablePredicateFilter( - true); + TableLoadOptions read_context_builder_load{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_read, + TableRead::Create(read_context_builder_load, std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index 8e7ba94fa..e29984fa3 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -35,6 +35,7 @@ #include "arrow/ipc/json_simple.h" #include "arrow/util/string_builder.h" #include "gtest/gtest.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/file_index/bitmap/bitmap_file_index_factory.h" @@ -142,16 +143,19 @@ class WriteAndReadInteWithIndexTest const std::vector> splits, const std::shared_ptr& predicate, const std::shared_ptr& expected_array) const { - ReadContextBuilder context_builder(table_path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate) - .EnableLateMaterializing(false); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate).EnableLateMaterializing(false); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, + std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -1165,22 +1169,26 @@ class WriteAndReadInteWithIndexTest for (const auto& field : fields) { field_names.push_back(field->name()); } - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options["read.batch-size"] = "1"; + builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; builder.EnablePreciseFileIndexEvaluate(true) .EnablePredicateFilter(false) .SetReadFieldNames(field_names) .SetPredicate(predicate) .SetVectorSearch(vector_search) .SetFullTextSearch(full_text_search) - .AddOption("read.batch-size", "1") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") + .EnableLateMaterializing(false); if (enable_prefetch_) { builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -1279,16 +1287,22 @@ TEST_P(WriteAndReadInteWithIndexTest, TestSimple) { &expected_array); ASSERT_TRUE(array_status.ok()); - ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption("orc.read.enable-metrics", "true") + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder + .SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(batch_reader.get())); std::shared_ptr read_metrics = batch_reader->GetReaderMetrics(); @@ -1352,17 +1366,23 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); - ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .AddOption(Options::READ_BATCH_SIZE, "1") - .AddOption("orc.read.enable-metrics", "true") + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options[Options::FILE_FORMAT] = "orc"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder_options[Options::READ_BATCH_SIZE] = "1"; + context_builder_options["orc.read.enable-metrics"] = "true"; + context_builder + .SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); // simulate read limits, only read 3 batches std::vector batches; @@ -1636,17 +1656,20 @@ TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); - ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate) - .EnablePredicateFilter(true) - .EnableLateMaterializing(true); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate).EnablePredicateFilter(true).EnableLateMaterializing( + true); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(context_builder_load_options, std::move(read_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(std::vector>{split})); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -2323,19 +2346,23 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { auto predicate = PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, {Literal(10l), Literal(int64_t{2147483648})}); - ReadContextBuilder builder(path); + TableLoadOptions builder_load_options{path}; + std::map builder_options; + ReadContextBuilder builder; + builder_options["read.batch-size"] = "2"; + builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; builder.EnablePreciseFileIndexEvaluate(true) .EnablePredicateFilter(false) .SetPredicate(predicate) - .AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") + .EnableLateMaterializing(false); if (enable_prefetch_) { builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + TableRead::Create(builder_load_options, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -3172,15 +3199,18 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - ReadContextBuilder context_builder(path); - context_builder.AddOption("read.batch-size", "2") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") - .SetPredicate(predicate); + TableLoadOptions context_builder_load_options{path}; + std::map context_builder_options; + ReadContextBuilder context_builder; + context_builder_options["read.batch-size"] = "2"; + context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; + context_builder.SetPredicate(predicate); if (enable_prefetch_) { context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - Result> table_read = TableRead::Create(std::move(read_context)); + Result> table_read = TableRead::Create( + context_builder_load_options, std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(split); CHECK_HOOK_STATUS(batch_reader.status(), i); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 31a325ec1..d10319510 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -41,6 +41,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table_load_options.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_array.h" #include "paimon/common/data/binary_array_writer.h" @@ -289,11 +290,16 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface const std::vector>& data_splits, const std::string& blob_field, const std::vector>& expected_blobs) const { - ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadFieldNames({blob_field}); + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options = options; + read_context_builder.SetReadFieldNames({blob_field}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(data_splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -349,11 +355,15 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface const std::vector>& commit_messages, bool ignore_empty_commit = true, int64_t commit_identifier = BATCH_WRITE_COMMIT_IDENTIFIER) const { - CommitContextBuilder commit_builder(table_path, "commit_user_1"); + TableLoadOptions commit_builder_load_options{table_path}; + std::map commit_builder_options; + CommitContextBuilder commit_builder("commit_user_1"); commit_builder.IgnoreEmptyCommit(ignore_empty_commit); PAIMON_ASSIGN_OR_RAISE(auto commit_context, commit_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + PAIMON_ASSIGN_OR_RAISE( + auto file_store_commit, + FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + commit_builder_options)); return file_store_commit->Commit(commit_messages, commit_identifier); } @@ -2015,11 +2025,14 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { ASSERT_EQ(3, batches_2.size()); // write data - WriteContextBuilder context_builder(root_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{root_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); - Result> write = - FileStoreWrite::Create(std::move(write_context)); + context_builder.WithStreamingMode(true).Finish()); + Result> write = FileStoreWrite::Create( + context_builder_load_options, std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); // round 1 @@ -2451,12 +2464,15 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); - Result> write = - FileStoreWrite::Create(std::move(write_context)); + Result> write = FileStoreWrite::Create( + context_builder_load_options, std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -2476,14 +2492,16 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 400; i += paimon::test::RandomNumber(5, 15)) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(false) - .Finish()); - auto commit = FileStoreCommit::Create(std::move(commit_context)); + TableLoadOptions commit_context_builder_load_options{table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + auto commit = + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); CHECK_HOOK_STATUS(committed.status(), i); @@ -2545,12 +2563,15 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) { /*partition_keys=*/{}, /*primary_keys=*/{}, options)); - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ::ArrowArray c_array; ASSERT_TRUE(arrow::ExportArray(*src_array, &c_array).ok()); RecordBatchBuilder batch_builder(&c_array); @@ -2615,12 +2636,15 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { /*table_name=*/"append_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); - std::string root_path = write_context->GetRootPath(); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -2652,15 +2676,25 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { ASSERT_TRUE(file_exist); // check read result - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -2712,12 +2746,15 @@ TEST_P(WriteInteTest, TestPKTableWriteAndReadWithExternalPath) { /*table_name=*/"pk_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{"f0", "f1"}, options)); // write - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); - std::string root_path = write_context->GetRootPath(); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -2932,12 +2969,15 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { /*primary_keys=*/{}, options)); // write - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); - std::string root_path = write_context->GetRootPath(); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); auto write = [&](const std::string& json_array, const std::map& partition) -> void { std::shared_ptr array = @@ -2973,15 +3013,25 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { arrow::field("_VALUE_KIND", arrow::int8())); auto data_type = arrow::struct_(fields_with_row_kind); { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3020,18 +3070,28 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", " "}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3047,18 +3107,28 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", ""}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3074,18 +3144,28 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + std::vector> partition_filters = { {{"dt", "__DEFAULT_PARTITION__"}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), + scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + std::move(read_context), + read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3124,12 +3204,15 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*table_name=*/"bar", &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options)); // write - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).Finish()); - std::string root_path = write_context->GetRootPath(); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); + std::string root_path = context_builder_load_options.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -3155,15 +3238,25 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*commit_identifier=*/1)); // check read result - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::FILE_FORMAT, file_format); + TableLoadOptions scan_context_builder_load_options{table_path}; + std::map scan_context_builder_options; + ScanContextBuilder scan_context_builder; + scan_context_builder_options[Options::FILE_FORMAT] = file_format; + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, + TableScan::Create(scan_context_builder_load_options, + std::move(scan_context), scan_context_builder_options)); + + TableLoadOptions read_context_builder_load_options{table_path}; + std::map read_context_builder_options; + ReadContextBuilder read_context_builder; + read_context_builder_options[Options::FILE_FORMAT] = file_format; - ReadContextBuilder read_context_builder(table_path); - read_context_builder.AddOption(Options::FILE_FORMAT, file_format); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, + TableRead::Create(read_context_builder_load_options, + std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -3220,12 +3313,15 @@ TEST_P(WriteInteTest, TestWriteWithIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); - Result> write = - FileStoreWrite::Create(std::move(write_context)); + Result> write = FileStoreWrite::Create( + context_builder_load_options, std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -3273,11 +3369,15 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write data - WriteContextBuilder context_builder(table_path, "commit_user_1"); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithStreamingMode(true).Finish()); + context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); ASSERT_OK(file_store_write->Write(std::move(batch_0))); @@ -3293,14 +3393,16 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - CommitContextBuilder commit_context_builder(tmp_table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr commit_context, - commit_context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(false) - .Finish()); - auto commit = FileStoreCommit::Create(std::move(commit_context)); + TableLoadOptions commit_context_builder_load_options{tmp_table_path}; + std::map commit_context_builder_options; + CommitContextBuilder commit_context_builder("commit_user_1"); + commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; + commit_context_builder_options[Options::FILE_SYSTEM] = "local"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.IgnoreEmptyCommit(false).Finish()); + auto commit = + FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); CHECK_HOOK_STATUS(committed.status(), i); @@ -3348,14 +3450,17 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) { /*primary_keys=*/{}, options)); std::shared_ptr write_pool = GetMemoryPool(); { - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithMemoryPool(write_pool) - .WithStreamingMode(true) - .Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + context_builder.WithMemoryPool(write_pool).WithStreamingMode(true).Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); ASSERT_OK(file_store_write->Write(std::move(batch_0))); @@ -3532,11 +3637,15 @@ TEST_F(WriteInteTest, TestBranchWrite) { std::map options = {{Options::FILE_FORMAT, "orc"}, {Options::FILE_SYSTEM, "local"}}; - WriteContextBuilder context_builder(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options).WithBranch("rt").Finish()); + TableLoadOptions context_builder_load_options{table_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + context_builder_options["branch"] = "rt"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(context_builder_load_options, + std::move(write_context), context_builder_options)); // write to rt branch, pk table std::string data_0 = @@ -3577,11 +3686,15 @@ TEST_F(WriteInteTest, TestBranchWrite) { ASSERT_TRUE(expected_commit_message_0->TEST_Equal(*result_commit_msgs)); // write to main branch, append table - WriteContextBuilder context_builder1(table_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context1, - context_builder1.SetOptions(options).Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write1, - FileStoreWrite::Create(std::move(write_context1))); + TableLoadOptions context_builder1_load_options{table_path}; + std::map context_builder1_options; + WriteContextBuilder context_builder1("commit_user_1"); + context_builder1_options = options; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context1, context_builder1.Finish()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr file_store_write1, + FileStoreWrite::Create(context_builder1_load_options, std::move(write_context1), + context_builder1_options)); std::string data_1 = R"([["20240725", "watermelon", 3], ["20240725", "orange", 2] @@ -3715,11 +3828,14 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { auto write_array = [&](const std::vector& write_cols, const std::shared_ptr& write_array) -> Result>> { - WriteContextBuilder builder(table_path, commit_user); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + WriteContextBuilder builder(commit_user); builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + FileStoreWrite::Create(builder_load_options, + std::move(write_context), builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); auto record_batch = std::make_unique( @@ -3749,10 +3865,13 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { auto commit = [&](const std::vector>& commit_msgs, int64_t latest_snapshot_id, int64_t next_row_id) { - CommitContextBuilder builder(table_path, commit_user); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + CommitContextBuilder builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(std::move(commit_context))); + FileStoreCommit::Create(builder_load_options, + std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); auto commit_impl = dynamic_cast(file_store_commit.get()); @@ -3765,10 +3884,13 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { // first_row_id, write_cols, min_sequence_number, max_sequence_number using MetaType = std::tuple>, int64_t, int64_t>; auto check_committed_meta = [&](const std::vector& expected_meta) { - ScanContextBuilder builder(table_path); + TableLoadOptions builder_load_options{table_path}; + std::map builder_options; + ScanContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr table_scan, + TableScan::Create(builder_load_options, std::move(scan_context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto data_split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -4144,13 +4266,15 @@ TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) { /*primary_keys=*/{"id"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); + write_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - write_builder.SetOptions(options) - .WithStreamingMode(true) - .WithTempDirectory(tmp_dir) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN( auto batch_0, TestHelper::MakeRecordBatch( @@ -4210,10 +4334,14 @@ TEST_P(WriteInteTest, TestPkSpillableDiskQuotaExhaustedFallsBackToFlush) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto write_array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ ["Alice", 10, 1, 1.0], @@ -4266,10 +4394,14 @@ TEST_P(WriteInteTest, TestPkSpillableGlobalMemoryPreemptionDataCorrectness) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); // Write long strings to different partitions to trigger global memory preemption. std::string long_str_a(48, 'a'); @@ -4364,10 +4496,14 @@ TEST_P(WriteInteTest, TestPkSpillableTempFilesCleanedAfterPrepareCommit) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto batch1 = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([["Alice", 10, 1]])").ValueOrDie(); @@ -4428,10 +4564,14 @@ TEST_P(WriteInteTest, TestPkSpillableIntermediateMergeWithTempFileTracking) { std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto write_array_fn = [](FileStoreWrite* writer, const std::map& partition, int32_t bucket, @@ -4511,10 +4651,14 @@ TEST_P(WriteInteTest, TestPkSpillableVector) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); auto write_array = [](FileStoreWrite* writer, const std::shared_ptr& array) { ArrowArray c_array; @@ -4594,12 +4738,16 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { return writer->Write(std::move(batch)); }; - WriteContextBuilder write_builder(table_path, "commit_user_1"); + TableLoadOptions write_builder_load_options{table_path}; + std::map write_builder_options; + WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true) .WithTempDirectory(tmp_dir) .SetWriteBufferSpillThreadNumber(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(write_builder_load_options, + std::move(write_context), write_builder_options)); // Round 1: Bucket 0 writes Alice(dup) + Bob, Bucket 1 writes Dave(dup) + Eve auto r1_b0_batch1 = @@ -4787,14 +4935,17 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { // Write with spill enabled (WithTempDirectory) std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - WriteContextBuilder context_builder(root_path, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, - context_builder.SetOptions(options) - .WithStreamingMode(true) - .WithTempDirectory(tmp_dir) - .Finish()); - ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(std::move(write_context))); + TableLoadOptions context_builder_load_options{root_path}; + std::map context_builder_options; + WriteContextBuilder context_builder("commit_user_1"); + context_builder_options = options; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_context, + context_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); + ASSERT_OK_AND_ASSIGN( + auto file_store_write, + FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + context_builder_options)); // Start IOHook only for spill write/read operations. io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); From 28b352ff233005ee00374a130897918850a1b9ac Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 13 Nov 2026 21:26:16 +0800 Subject: [PATCH 2/6] fix(table): align copy validation and retain catalog branch schemas --- include/paimon/catalog/table.h | 2 + ...erge_tree_compact_manager_factory_test.cpp | 16 +- .../operation/file_store_commit_impl_test.cpp | 31 ++- .../core/operation/file_store_write_test.cpp | 3 +- src/paimon/core/schema/schema_validation.cpp | 102 ++++++++++ src/paimon/core/schema/schema_validation.h | 5 + .../core/schema/schema_validation_test.cpp | 84 ++++++++ src/paimon/core/table/file_store_table.cpp | 5 + .../core/table/source/table_read_test.cpp | 24 ++- .../table/system/audit_log_system_table.cpp | 4 +- .../core/table/system/binlog_system_table.cpp | 4 +- .../system/read_optimized_system_table.cpp | 4 +- src/paimon/core/table/system/system_table.cpp | 4 +- src/paimon/core/table/system/system_table.h | 2 +- src/paimon/core/table/table_factory.cpp | 30 +-- src/paimon/core/table/table_test.cpp | 180 +++++++++++++++++- src/paimon/rest/rest_catalog_test.cpp | 9 +- test/inte/data_evolution_table_test.cpp | 2 +- test/inte/scan_and_read_inte_test.cpp | 48 +++-- test/inte/scan_inte_test.cpp | 2 +- 20 files changed, 508 insertions(+), 53 deletions(-) diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 112c1010e..2b43a66d7 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -85,6 +85,8 @@ class PAIMON_EXPORT Table { /// Returns an independent table with dynamic options applied to its bound configuration. /// The original table is unchanged. Type, location, branch, catalog identity and storage /// bindings are retained; load another table to select a different branch. + /// Managed tables reject changes to immutable schema options (such as bucket-key), and + /// validate conditional option changes using the original table's configuration. virtual Result> Copy( const std::map& dynamic_options) const = 0; diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index 04fe07b1c..ea87d5e72 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -357,13 +357,15 @@ TEST_F(MergeTreeCompactManagerFactoryWriteTest, TEST_F(MergeTreeCompactManagerFactoryWriteTest, TestWriteShouldFailWhenFirstRowWithDeletionVectors) { - ASSERT_OK_AND_ASSIGN( - auto file_store_write, - CreateSingleStringFileStoreWrite({{"bucket", "1"}, {Options::MERGE_ENGINE, "first-row"}}, - /*with_io_manager=*/false, - {{Options::BUCKET, "1"}, - {Options::MERGE_ENGINE, "first-row"}, - {Options::DELETION_VECTORS_ENABLED, "true"}})); + // Permit the runtime DV change so this test reaches the first-row writer's own validation. + ASSERT_OK_AND_ASSIGN(auto file_store_write, CreateSingleStringFileStoreWrite( + {{"bucket", "1"}, + {Options::MERGE_ENGINE, "first-row"}, + {"deletion-vectors.modifiable", "true"}}, + /*with_io_manager=*/false, + {{Options::BUCKET, "1"}, + {Options::MERGE_ENGINE, "first-row"}, + {Options::DELETION_VECTORS_ENABLED, "true"}})); ASSERT_NOK_WITH_MSG(WriteSingleStringRow(file_store_write.get(), /*bucket=*/0, "k1"), "First row merge engine does not need deletion vectors because there " diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index 211a7fea9..eebc7cb81 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -43,6 +43,7 @@ #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/linked_hash_map.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" @@ -273,10 +274,25 @@ class FileStoreCommitImplTest : public testing::Test { return catalog; } + // These tests create a fresh temporary table before committing fixture messages. Declare + // layout options in its schema instead of changing them through a runtime commit option. + Status ConfigureTableOptions(const std::map& options) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr schema, + checked_pointer_cast(table_schema_)->MergeOptions(options)); + PAIMON_ASSIGN_OR_RAISE(std::string json, schema->GetJsonSchema()); + SchemaManager manager(file_system_, table_path_); + PAIMON_RETURN_NOT_OK( + file_system_->WriteFile(manager.ToSchemaPath(schema->Id()), json, true)); + table_schema_ = std::move(schema); + return Status::OK(); + } + std::shared_ptr TableSchemaWithId(int64_t schema_id) const { EXPECT_OK_AND_ASSIGN( std::unique_ptr evolved, - TableSchema::Create(schema_id, arrow::schema(fields_), {"f1"}, {}, {})); + TableSchema::Create(schema_id, arrow::schema(fields_), {"f1"}, {}, + checked_pointer_cast(table_schema_)->Options())); return std::shared_ptr(std::move(evolved)); } @@ -1765,6 +1781,7 @@ TEST_F(FileStoreCommitImplTest, TestAGivenFileSystemWinsOverTheCatalogs) { } TEST_F(FileStoreCommitImplTest, TestRealtimeCommitIsIdempotentThroughTheCatalog) { + ASSERT_OK(ConfigureTableOptions({{Options::BUCKET, "2"}})); ASSERT_OK(PrepareFirstCommitMessageFiles()); auto catalog = CreateCatalog(); catalog->CheckBaseSnapshotUuid(); @@ -1793,6 +1810,7 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitIsIdempotentThroughTheCatalog) } TEST_F(FileStoreCommitImplTest, TestCommitRestoresAlreadyCommittedRealtimeSnapshot) { + ASSERT_OK(ConfigureTableOptions({{Options::BUCKET, "2"}})); ASSERT_OK(PrepareFirstCommitMessageFiles()); auto catalog = CreateCatalog(); catalog->CheckBaseSnapshotUuid(); @@ -1831,6 +1849,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitRestoresAlreadyCommittedRealtimeSnapsh } TEST_F(FileStoreCommitImplTest, TestRealtimeCommitReportsAnUnpublishedHistory) { + ASSERT_OK(ConfigureTableOptions({{Options::BUCKET, "2"}})); ASSERT_OK(PrepareFirstCommitMessageFiles()); auto catalog = CreateCatalog(); catalog->SetHeldSnapshot(BuildTestSnapshot(7, "snapshot-uuid-7")); @@ -1893,6 +1912,8 @@ TEST_F(FileStoreCommitImplTest, TestRowIdSchemaPreCheckPassesOnAPublishedSchema) } TEST_F(FileStoreCommitImplTest, TestCommitRefusesARowIdCheckMissingItsSchema) { + ASSERT_OK(ConfigureTableOptions( + {{Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::ROW_TRACKING_ENABLED, "true"}})); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); std::shared_ptr catalog = CreateCatalogServingSchemaId(7); @@ -1915,6 +1936,8 @@ TEST_F(FileStoreCommitImplTest, TestCommitRefusesARowIdCheckMissingItsSchema) { } TEST_F(FileStoreCommitImplTest, TestRowIdChecksUseCatalogLatestAndPublishedHistory) { + ASSERT_OK(ConfigureTableOptions( + {{Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::ROW_TRACKING_ENABLED, "true"}})); std::shared_ptr catalog = CreateCatalogServingSchemaId(7); catalog->CheckBaseSnapshotUuid(); CatalogCommitSpec spec; @@ -1954,6 +1977,8 @@ TEST_F(FileStoreCommitImplTest, TestRowIdChecksUseCatalogLatestAndPublishedHisto } TEST_F(FileStoreCommitImplTest, TestRowIdConflictChecksSnapshotBoundsAndHistory) { + ASSERT_OK(ConfigureTableOptions( + {{Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::ROW_TRACKING_ENABLED, "true"}})); CatalogCommitSpec spec; spec.catalog = CreateCatalog(); spec.options = {{Options::DATA_EVOLUTION_ENABLED, "true"}, @@ -2196,6 +2221,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { } TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) { + ASSERT_OK(ConfigureTableOptions({{Options::BUCKET, "2"}})); const std::vector data_files = { "/f1=10/bucket-0/data-51a45441-6037-4af3-b67b-5cefd75dc6f2-0.orc", "/f1=10/bucket-1/data-6828284c-e707-49b5-af6b-69be79af120c-0.orc", @@ -3856,6 +3882,7 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { } TEST_F(FileStoreCommitImplTest, TestCollectChanges) { + ASSERT_OK(ConfigureTableOptions({{Options::BUCKET, "10"}})); TableLoadOptions context_builder_load_options{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); @@ -4239,6 +4266,8 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { + ASSERT_OK(ConfigureTableOptions( + {{Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::ROW_TRACKING_ENABLED, "true"}})); TableLoadOptions context_builder_load_options{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 8ba2b0ab5..4ce2e7341 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -436,7 +436,7 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidBucket) { ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, std::move(write_context), context_builder_options), - "not support bucket -2 in append table"); + "Cannot change bucket from or to -1 dynamically"); } TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidWriteType) { @@ -460,7 +460,6 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidWriteType) { TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); - context_builder_options[Options::BUCKET] = "-1"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.WithWriteSchema({"field_non_exist"}).Finish()); ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index b8319f6cf..8312ed76c 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -227,6 +227,108 @@ Status CheckOptionsConflict(const std::map& options, } // namespace +Result> SchemaValidation::MergeDynamicOptions( + const TableSchema& schema, const std::map& dynamic_options) { + // Java CoreOptions.IMMUTABLE_OPTIONS. Keep even options not interpreted by this reader: + // schemas written by another engine may contain them, and Copy must preserve their layout. + static const std::unordered_set kImmutableOptions = { + "type", + "bucket-key", + "bucket-function.type", + "bucket-function.provider", + "data-file.path-directory", + "merge-engine", + "sequence.snapshot-ordering", + "aggregation.remove-record-on-delete", + "partial-update.remove-record-on-delete", + "partial-update.remove-record-on-sequence-group", + "rowkind.field", + "primary-key", + "primary-key.nullable", + "partition", + "dynamic-bucket.initial-buckets", + "force-lookup", + "row-tracking.enabled", + "data-evolution.enabled", + "index-file-in-data-file-dir", + "blob-field", + "video-frame-field", + "blob-descriptor-field", + "blob-view-field", + "pk-clustering-override"}; + const auto& old_options = schema.Options(); + PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] bool query_auth, + QueryAuthReadEnabled(old_options, dynamic_options)); + auto options = old_options; + for (const auto& [key, value] : dynamic_options) { + auto old = old_options.find(key); + if (old != old_options.end() && old->second == value) { + continue; + } + if (key == Options::TYPE && + StringUtils::ToLowerCase(value) == + StringUtils::ToLowerCase(old == old_options.end() ? "table" : old->second)) { + continue; + } + if (old == old_options.end() && (key == "primary-key" || key == "partition")) { + std::vector keys; + for (auto field : StringUtils::Split(value, ",")) { + StringUtils::Trim(&field); + if (!field.empty()) { + keys.push_back(std::move(field)); + } + } + if (keys == (key == "primary-key" ? schema.PrimaryKeys() : schema.PartitionKeys())) { + continue; + } + } + if (kImmutableOptions.count(key)) { + return Status::Invalid( + fmt::format("Cannot change immutable option '{}' dynamically", key)); + } + if (key == Options::BUCKET) { + PAIMON_ASSIGN_OR_RAISE(int32_t old_bucket, + OptionsUtils::GetValueFromMap(old_options, key, -1)); + PAIMON_ASSIGN_OR_RAISE(int32_t new_bucket, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_bucket == -1 || new_bucket == -1) { + return Status::Invalid("Cannot change bucket from or to -1 dynamically"); + } + } + if (key == Options::DELETION_VECTORS_ENABLED) { + PAIMON_ASSIGN_OR_RAISE(bool modifiable, + OptionsUtils::GetValueFromMap( + old_options, "deletion-vectors.modifiable", false)); + PAIMON_ASSIGN_OR_RAISE(bool old_enabled, + OptionsUtils::GetValueFromMap(old_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_enabled, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (!modifiable && old_enabled != new_enabled) { + return Status::Invalid( + "Cannot change deletion vectors mode unless deletion-vectors.modifiable " + "was already enabled"); + } + } + if (key == Options::IGNORE_DELETE || key == "ignore-update-before" || + key == "data-evolution.nested-field.enabled") { + PAIMON_ASSIGN_OR_RAISE(bool old_enabled, + OptionsUtils::GetValueFromMap(old_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_enabled, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_enabled && !new_enabled) { + return Status::Invalid( + fmt::format("Cannot disable table option '{}' dynamically", key)); + } + } + if (key == "clustering.columns" && old_options.count(Options::PK_CLUSTERING_OVERRIDE)) { + return Status::Invalid( + "Cannot change clustering.columns when pk-clustering-override is set"); + } + options[key] = value; + } + return options; +} + bool SchemaValidation::IsComplexType(const std::shared_ptr& field) { arrow::Type::type arrow_type_id = field->type()->id(); return (arrow_type_id == arrow::Type::TIMESTAMP || arrow_type_id == arrow::Type::DECIMAL128 || diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 0ad605c5a..182f8184b 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -52,6 +52,11 @@ class SchemaValidation { static Status ValidateTableSchema(const TableSchema& schema); + /// Merges runtime options using Java's Copy immutability rules. Normalized primary-key, + /// partition and type no-ops retain their original representation in the schema. + static Result> MergeDynamicOptions( + const TableSchema& schema, const std::map& dynamic_options); + /// Validates real-time options after operation-specific options have been merged with the /// table schema options. static Status ValidateRealtime(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 4694be20b..3f7bb3b5f 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -2158,4 +2158,88 @@ TEST(SchemaValidationTest, TestMapSharedShreddingRejectsPostponeBucketMode) { "MAP shared-shredding currently does not support postpone bucket mode."); } +TEST(SchemaValidationTest, TestCopyImmutableOptions) { + const std::map> changes = { + {"bucket-key", {"a", "b"}}, + {"bucket-function.type", {"default", "mod"}}, + {"bucket-function.provider", {"original", "other"}}, + {"merge-engine", {"deduplicate", "partial-update"}}, + {"row-tracking.enabled", {"false", "true"}}, + {"data-evolution.enabled", {"false", "true"}}, + {"sequence.snapshot-ordering", {"false", "true"}}, + {"blob-field", {"a", "b"}}, + {"primary-key.nullable", {"false", "true"}}, + {"force-lookup", {"false", "true"}}}; + auto arrow_schema = arrow::schema( + {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::int32(), false)}); + for (const auto& [key, values] : changes) { + SCOPED_TRACE(key); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr schema, + TableSchema::Create(0, arrow_schema, {}, {"a", "b"}, {{key, values.first}})); + ASSERT_OK_AND_ASSIGN(auto unchanged, + SchemaValidation::MergeDynamicOptions(*schema, {{key, values.first}})); + ASSERT_EQ(unchanged, schema->Options()); + ASSERT_NOK_WITH_MSG(SchemaValidation::MergeDynamicOptions(*schema, {{key, values.second}}), + "immutable option"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr unset, + TableSchema::Create(0, arrow_schema, {}, {"a", "b"}, {})); + // Like Java, explicitly setting an immutable default is a change when it was absent. + ASSERT_NOK_WITH_MSG(SchemaValidation::MergeDynamicOptions(*unset, {{key, values.first}}), + "immutable option"); + } +} + +TEST(SchemaValidationTest, TestCopyConditionalOptionChanges) { + using OptionsMap = std::map; + struct TestCase { + OptionsMap old_options; + OptionsMap dynamic_options; + bool allowed; + }; + const std::vector cases = { + {{{"bucket", "2"}}, {{"bucket", "4"}}, true}, + {{{"bucket", "2"}}, {{"bucket", "-1"}}, false}, + {{{"bucket", "-1"}}, {{"bucket", "2"}}, false}, + {{}, {{"bucket", "2"}}, false}, + {{{"bucket", "-1"}}, {{"bucket", "-1"}}, true}, + {{{"bucket", "2"}}, {{"bucket", "invalid"}}, false}, + {{}, {{"deletion-vectors.enabled", "false"}}, true}, + {{}, {{"deletion-vectors.enabled", "true"}}, false}, + {{{"deletion-vectors.enabled", "true"}}, {{"deletion-vectors.enabled", "false"}}, false}, + {{}, + {{"deletion-vectors.enabled", "true"}, {"deletion-vectors.modifiable", "true"}}, + false}, + {{{"deletion-vectors.modifiable", "true"}}, {{"deletion-vectors.enabled", "true"}}, true}, + {{{"deletion-vectors.modifiable", "true"}, {"deletion-vectors.enabled", "true"}}, + {{"deletion-vectors.enabled", "false"}}, + true}, + {{}, {{"ignore-delete", "true"}}, true}, + {{{"ignore-delete", "true"}}, {{"ignore-delete", "false"}}, false}, + {{}, {{"ignore-update-before", "true"}}, true}, + {{{"ignore-update-before", "true"}}, {{"ignore-update-before", "false"}}, false}, + {{}, {{"data-evolution.nested-field.enabled", "true"}}, true}, + {{{"data-evolution.nested-field.enabled", "true"}}, + {{"data-evolution.nested-field.enabled", "false"}}, + false}, + {{}, {{"clustering.columns", "a"}}, true}, + {{{"pk-clustering-override", "false"}}, {{"clustering.columns", "a"}}, false}, + {{{"pk-clustering-override", "true"}, {"clustering.columns", "a"}}, + {{"clustering.columns", "a"}}, + true}, + {{{"query-auth.enabled", "true"}}, {{"query-auth.enabled", "false"}}, false}, + }; + auto arrow_schema = arrow::schema({arrow::field("a", arrow::int32(), false)}); + for (size_t i = 0; i < cases.size(); ++i) { + SCOPED_TRACE(i); + const auto& test_case = cases[i]; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr schema, + TableSchema::Create(0, arrow_schema, {}, {"a"}, test_case.old_options)); + auto result = SchemaValidation::MergeDynamicOptions(*schema, test_case.dynamic_options); + ASSERT_EQ(result.ok(), test_case.allowed) << result.status().ToString(); + ASSERT_EQ(schema->Options(), test_case.old_options); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 49323b386..42e253a27 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -322,6 +322,11 @@ Result> FileStoreTable::Copy( PAIMON_ASSIGN_OR_RAISE(OptionsMap options, ResolveOptionMap(dynamic_options)); PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] bool query_auth, SchemaValidation::QueryAuthReadEnabled(options_, dynamic_options)); + // Static factories have already resolved some options. Validate against the loaded schema, + // before those overrides, so that both entry points enforce the same immutability rules. + PAIMON_ASSIGN_OR_RAISE(options, SchemaValidation::MergeDynamicOptions( + *checked_pointer_cast(schema_), options)); + options["path"] = root_path_; auto explicit_options = dynamic_options_; for (const auto& [key, value] : dynamic_options) { explicit_options[key] = value; diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index e344a4a46..68c09eeb6 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -413,6 +413,15 @@ TEST(TableReadTest, TestPreciseScalarEvaluationRejectsUnsupportedTableModes) { } else { builder_options[Options::DATA_EVOLUTION_ENABLED] = "true"; } + // Declare the unsupported mode in the loaded schema, rather than changing an immutable + // layout option dynamically, so the test reaches precise-evaluation validation. + SchemaManager manager(std::make_shared(), path); + ASSERT_OK_AND_ASSIGN(std::optional> latest, manager.Latest()); + ASSERT_TRUE(latest); + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + latest.value()->MergeOptions(builder_options)); + ASSERT_OK_AND_ASSIGN(builder_load_options.table_schema_json, schema->GetJsonSchema()); + builder_options.clear(); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); const std::string expected_error = primary_key ? "Precise file-index evaluation on primary-key tables requires " @@ -476,7 +485,7 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { } TEST(TableReadTest, TestMergeOptions) { - std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + std::string path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; TableLoadOptions context_builder_load_options{path}; std::map context_builder_options; ReadContextBuilder context_builder; @@ -487,8 +496,6 @@ TEST(TableReadTest, TestMergeOptions) { context_builder_options["bucket"] = "10"; - context_builder_options["bucket-key"] = "f3"; - ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(context_builder_load_options, std::move(read_context), @@ -500,9 +507,13 @@ TEST(TableReadTest, TestMergeOptions) { ASSERT_TRUE(typed_split_read); const auto& core_options = typed_split_read->options_; std::map expected_options = { - {"read.batch-size", "2"}, {"orc.read.enable-lazy-decoding", "true"}, - {"bucket", "10"}, {"bucket-key", "f3"}, - {"manifest.format", "orc"}, {"file.format", "orc"}}; + {"read.batch-size", "2"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"bucket", "10"}, + {"bucket-key", "f2"}, + {"manifest.format", "orc"}, + {"file.format", "orc"}, + {"path", path}}; ASSERT_EQ(expected_options, core_options.ToMap()); } @@ -536,6 +547,7 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { auto dependency_context = internal_context->CreateDependencyCatalogContext(); auto expected_options = table_options; + expected_options["path"] = builder_load_options.path; expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index d6da1bc37..c408e1623 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -522,9 +522,11 @@ Result> AuditLogSystemTable::ReadOptions( Result> AuditLogSystemTable::Copy( const std::map& dynamic_options) const { using OptionsMap = std::map; - PAIMON_ASSIGN_OR_RAISE(OptionsMap options, CopyOptions(options_, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(OptionsMap options, + CopyOptions(*table_schema_, options_, dynamic_options)); auto table = std::make_shared(*this); table->options_ = std::move(options); + PAIMON_ASSIGN_OR_RAISE(table->table_schema_, table_schema_->Copy(table->options_)); PAIMON_RETURN_NOT_OK(table->AdoptOwnSchema()); return std::shared_ptr
(std::move(table)); } diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 316d6dbcc..2348b4fc9 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -107,9 +107,11 @@ Result> BinlogSystemTable::NewRead( Result> BinlogSystemTable::Copy( const std::map& dynamic_options) const { using OptionsMap = std::map; - PAIMON_ASSIGN_OR_RAISE(OptionsMap options, CopyOptions(options_, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(OptionsMap options, + CopyOptions(*table_schema_, options_, dynamic_options)); auto table = std::make_shared(*this); table->options_ = std::move(options); + PAIMON_ASSIGN_OR_RAISE(table->table_schema_, table_schema_->Copy(table->options_)); PAIMON_RETURN_NOT_OK(table->AdoptOwnSchema()); return std::shared_ptr
(std::move(table)); } diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 90d28303c..a48c13ed6 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -140,9 +140,11 @@ Result> ReadOptimizedSystemTable::NewRead( Result> ReadOptimizedSystemTable::Copy( const std::map& dynamic_options) const { using OptionsMap = std::map; - PAIMON_ASSIGN_OR_RAISE(OptionsMap options, CopyOptions(options_, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(OptionsMap options, + CopyOptions(*table_schema_, options_, dynamic_options)); auto table = std::make_shared(*this); table->options_ = std::move(options); + PAIMON_ASSIGN_OR_RAISE(table->table_schema_, table_schema_->Copy(table->options_)); PAIMON_RETURN_NOT_OK(table->AdoptOwnSchema()); return std::shared_ptr
(std::move(table)); } diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index aaa1029f5..bd76cedac 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -247,7 +247,7 @@ Result> SystemTable::CreateDataTable( } Result> SystemTable::CopyOptions( - const std::map& bound_options, + const TableSchema& table_schema, const std::map& bound_options, const std::map& overrides) { auto options = bound_options; for (const auto& [key, value] : overrides) { @@ -266,7 +266,7 @@ Result> SystemTable::CopyOptions( if (type != bound_type) { return Status::Invalid("copy type contradicts the loaded table"); } - return options; + return SchemaValidation::MergeDynamicOptions(table_schema, options); } Result> SystemTable::NewWrite( diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 65dc1a9a0..227ccd8e4 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -92,7 +92,7 @@ class SystemTable : public Table, public std::enable_shared_from_this> CopyOptions( - const std::map& bound_options, + const TableSchema& table_schema, const std::map& bound_options, const std::map& overrides); /// Builds the data table using this view's data schema and options. diff --git a/src/paimon/core/table/table_factory.cpp b/src/paimon/core/table/table_factory.cpp index cdf92d08f..37b9bb649 100644 --- a/src/paimon/core/table/table_factory.cpp +++ b/src/paimon/core/table/table_factory.cpp @@ -126,6 +126,7 @@ Result> TableFactory::Create( std::optional table_id = load.table_id; std::optional schema_json = load.table_schema_json; std::shared_ptr file_system = load.file_system; + std::shared_ptr catalog_schema; // A catalog-only load obtains identity, schema and location from one table generation. if (load.catalog && (path.empty() || load.table_id)) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, load.catalog->GetTable(*identifier)); @@ -135,8 +136,10 @@ Result> TableFactory::Create( } if (path.empty()) { path = table->GetRootPath(); - if (!schema_json) { - PAIMON_ASSIGN_OR_RAISE(schema_json, table->GetSchema()->GetJsonSchema()); + if (!schema_json || !BranchManager::IsMainBranch(branch)) { + // The catalog has resolved this branch's schema. table_schema_json is a + // main-only input and must not discard it or substitute another branch's schema. + catalog_schema = std::dynamic_pointer_cast(table->GetSchema()); } } if (!file_system && fs_scheme_map.empty()) { @@ -158,7 +161,8 @@ Result> TableFactory::Create( if (load.catalog && !file_system && fs_scheme_map.empty()) { PAIMON_ASSIGN_OR_RAISE(file_system, load.catalog->GetTableFileSystem(*identifier)); } - if (writing && load.catalog && BranchManager::IsMainBranch(branch) && !schema_json) { + if (writing && load.catalog && BranchManager::IsMainBranch(branch) && !schema_json && + !catalog_schema) { PAIMON_ASSIGN_OR_RAISE(Identifier identifier, CatalogUtils::BranchIdentifier(*load.identifier, branch)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, @@ -185,12 +189,15 @@ Result> TableFactory::Create( fs_scheme_map, schema_json)); return std::shared_ptr
(std::move(table)); } - std::shared_ptr schema; + std::shared_ptr schema = catalog_schema; const std::string metadata_branch = branch; auto schema_manager = std::make_shared(temporary.GetFileSystem(), path, branch); - Result> format_result = - FormatTableLoader::TryLoad(temporary.GetFileSystem(), path, branch, dynamic_options, - schema_json, schema_manager.get(), &schema); + Result> format_result = std::shared_ptr(); + if (!schema) { + format_result = + FormatTableLoader::TryLoad(temporary.GetFileSystem(), path, branch, dynamic_options, + schema_json, schema_manager.get(), &schema); + } if (load.catalog && BranchManager::IsMainBranch(branch) && (!format_result.ok() || format_result.value() != nullptr || !schema)) { // An external format table may have real data under `schema`, including file names @@ -220,10 +227,11 @@ Result> TableFactory::Create( return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } if (!writing) { - PAIMON_ASSIGN_OR_RAISE(dynamic_options, CatalogUtils::ResolveQueryAuthReadOptions( - load.catalog, identifier, dynamic_options, - branch, temporary.GetFileSystem(), path, - schema_json.has_value(), operation)); + PAIMON_ASSIGN_OR_RAISE( + dynamic_options, + CatalogUtils::ResolveQueryAuthReadOptions( + load.catalog, identifier, dynamic_options, branch, temporary.GetFileSystem(), path, + schema_json.has_value() || catalog_schema, operation)); } auto merged = schema->Options(); for (const auto& [key, value] : dynamic_options) { diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index c97752a5e..cf7553310 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -88,6 +88,8 @@ #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/data_generator.h" #include "paimon/testing/utils/deletion_vector_test_helper.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" @@ -174,6 +176,119 @@ TEST(TableTest, TestCopyRetainsBindingsAndMergesOptionsIndependently) { ASSERT_TRUE(weak_catalog.expired()); } +TEST(TableTest, TestCopyRejectsBucketKeyChangesWithoutLosingRows) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto pool = GetDefaultPool(); + const std::map options = { + {Options::BUCKET, "2"}, {Options::BUCKET_KEY, "a"}, {Options::FILE_FORMAT, "parquet"}}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr helper, + TestHelper::Create(dir->Str(), + arrow::schema({arrow::field("a", arrow::int32(), false), + arrow::field("b", arrow::int32(), false)}), + {}, {"a", "b"}, options, false)); + ASSERT_OK_AND_ASSIGN(std::optional> schema, + helper->LatestSchema()); + ASSERT_TRUE(schema); + const auto original_options = schema.value()->Options(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(dir->GetFileSystem(), dir->Str() + "/foo.db/bar", schema.value(), + Identifier("foo", "bar"), "", "main")); + DataGenerator generator(schema.value(), pool); + // Under the default hash, a=0 belongs to bucket 1 while b=1 belongs to bucket 0. + auto row = + BinaryRowGenerator::GenerateRow(RowKind::Insert(), {int32_t{0}, int32_t{1}}, pool.get()); + ASSERT_OK_AND_ASSIGN(std::vector> batches, + generator.SplitArrayByPartitionAndBucket({row})); + ASSERT_OK(helper->WriteAndCommit(std::move(batches), 0, std::nullopt)); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "b"}}), "immutable option 'bucket-key'"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
copied, + table->Copy({{Options::BUCKET_KEY, "a"}, {Options::READ_BATCH_SIZE, "1"}})); + ASSERT_NOK_WITH_MSG(copied->Copy({{Options::BUCKET_KEY, "b"}}), + "immutable option 'bucket-key'"); + ASSERT_EQ(schema.value()->Options(), original_options); + auto predicate = PredicateBuilder::Equal(1, "b", FieldType::INT, Literal(int32_t{1})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + ScanContextBuilder().SetPredicate(predicate).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, copied->NewScan(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + // A static factory must validate against the persisted schema, before merging overrides. + TableLoadOptions load{table->GetRootPath()}; + const std::map invalid = {{Options::BUCKET_KEY, "b"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(load, std::move(scan_context), invalid), "bucket-key"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(load, std::move(read_context), invalid), "bucket-key"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + WriteContextBuilder("writer").Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(load, std::move(write_context), invalid), + "bucket-key"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + CommitContextBuilder("writer").Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(load, std::move(commit_context), invalid), + "bucket-key"); +} + +TEST(TableTest, TestCopyNormalizesSchemaKeysAndRetainsPath) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(7, + arrow::schema({arrow::field("a", arrow::int32(), false), + arrow::field("b", arrow::int32(), false)}), + {"a"}, {"a", "b"}, {{Options::BUCKET, "2"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, + Identifier("db", "tbl"), "uuid", "main")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{"primary-key", " a, , b "}, + {"partition", "a, "}, + {"type", "TABLE"}, + {"path", "another/table"}, + {Options::READ_BATCH_SIZE, "32"}})); + auto copied_schema = checked_pointer_cast(copied->GetSchema()); + ASSERT_EQ(copied_schema->Id(), 7); + ASSERT_EQ(copied_schema->Options().count("primary-key"), 0u); + ASSERT_EQ(copied_schema->Options().count("partition"), 0u); + ASSERT_EQ(copied_schema->Options().count("type"), 0u); + ASSERT_EQ(copied_schema->Options().at("path"), table->GetRootPath()); + ASSERT_EQ(copied_schema->Options().at(Options::READ_BATCH_SIZE), "32"); + ASSERT_NOK_WITH_MSG(copied->Copy({{"primary-key", "b,a"}}), "primary-key"); + ASSERT_NOK_WITH_MSG(copied->Copy({{"partition", "b"}}), "partition"); + ASSERT_EQ(schema->Options().count(Options::READ_BATCH_SIZE), 0u); +} + +TEST(TableTest, TestSystemTableCopiesPreserveManagedOptionRules) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + SchemaManager manager(dir->GetFileSystem(), dir->Str()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + manager.CreateTable(arrow::schema({arrow::field("a", arrow::int32(), false), + arrow::field("b", arrow::int32(), false)}), + {}, {"a", "b"}, {{Options::BUCKET, "2"}, {Options::BUCKET_KEY, "a"}})); + for (const std::string kind : {"ro", "audit_log", "binlog"}) { + SCOPED_TRACE(kind); + TableLoadOptions load{dir->Str() + "$" + kind}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(load, {}, "read")); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "b"}}), "bucket-key"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::IGNORE_DELETE, "true"}})); + ASSERT_NOK_WITH_MSG(copied->Copy({{Options::IGNORE_DELETE, "false"}}), "ignore-delete"); + ASSERT_OK(table->Copy({{Options::IGNORE_DELETE, "false"}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ScanContextBuilder().Finish()); + ASSERT_NOK_WITH_MSG( + TableScan::Create(load, std::move(context), {{Options::BUCKET_KEY, "b"}}), + "bucket-key"); + } +} + TEST(TableTest, TestGetTableWithUnknownDatabase) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -2495,7 +2610,7 @@ TEST(TableTest, TestOperationOptionsDoNotMutateTheLoadedTable) { table->Copy(invalid_context_builder_options)); return configured->NewScan(std::move(invalid_context)); })(), - "bucket=0"); + "Cannot change bucket from or to -1 dynamically"); ASSERT_EQ(options, schema->Options()); std::map context_builder_27781_options; ScanContextBuilder context_builder_27781; @@ -3533,6 +3648,69 @@ TEST(TableTest, TestPreloadedSchemaIsIndependentOfReadProjection) { table->NewRead(std::move(context))); } +TEST(TableTest, TestCatalogOnlyLoadsRetainTheResolvedBranchSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr remote_schema, + TableSchema::Create(7, arrow::schema({arrow::field("remote", arrow::int32())}), {}, {}, + {})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr main_schema, + TableSchema::Create(0, arrow::schema({arrow::field("main_only", arrow::int32())}), {}, {}, + {})); + for (const std::string branch : {"main", "dev"}) { + for (bool stale_disk_schema : {false, true}) { + SCOPED_TRACE(branch); + SCOPED_TRACE(stale_disk_schema); + const std::string path = + dir->Str() + "/" + branch + (stale_disk_schema ? "-stale" : "-empty"); + Identifier identifier("db", "tbl", branch); + if (stale_disk_schema) { + SchemaManager manager(fs, path, branch); + ASSERT_OK(manager.CreateTable(arrow::schema({arrow::field("stale", arrow::utf8())}), + {}, {}, {})); + } + ASSERT_OK_AND_ASSIGN( + std::shared_ptr original, + FileStoreTable::Create(fs, path, remote_schema, identifier, "uuid", branch)); + auto catalog = std::make_shared(); + catalog->SetTable(original); + catalog->SetTableLocation(path); + TableLoadOptions load; + load.catalog = catalog; + load.identifier.emplace(identifier); + load.table_id = "uuid"; + if (branch != "main") { + // A main schema supplied for fallback must not replace the catalog's branch schema. + ASSERT_OK_AND_ASSIGN(load.table_schema_json, main_schema->GetJsonSchema()); + } + for (const char* operation : {"read", "scan", "write", "commit"}) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + TableFactory::Create(load, {}, operation)); + ASSERT_EQ(checked_pointer_cast(loaded->GetSchema())->Id(), 7); + ASSERT_EQ(loaded->GetSchema()->FieldNames(), (std::vector{"remote"})); + ASSERT_EQ(loaded->GetRootPath(), path); + ASSERT_EQ(loaded->GetFileSystem(), fs); + ASSERT_EQ(loaded->CatalogUuid(), original->CatalogUuid()); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder().SetReadFieldNames({"remote"}).Finish()); + ASSERT_OK(TableRead::Create(load, std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder().Finish()); + ASSERT_OK(TableScan::Create(load, std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + WriteContextBuilder("writer").Finish()); + ASSERT_OK(FileStoreWrite::Create(load, std::move(write_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + CommitContextBuilder("writer").Finish()); + ASSERT_OK(FileStoreCommit::Create(load, std::move(commit_context))); + } + } +} + TEST(TableTest, TestLoadOptionsValidateExpectedCatalogUuid) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index aa8ce0ec1..f3d48ca24 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -1203,7 +1203,7 @@ TEST_F(RestCatalogTest, SystemTableSchema) { } TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, CreateRestCatalog()); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); @@ -1241,6 +1241,13 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_EQ(3, schema->Id()); ASSERT_EQ("b1", schema->Options().at(Options::BRANCH)); + TableLoadOptions load; + load.catalog = catalog; + load.identifier.emplace(branch_identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder().SetReadFieldNames({"b0"}).Finish()); + ASSERT_OK(TableRead::Create(load, std::move(read_context))); + // the default branch is addressed as the bare table: "t1$branch_main" resolves to // "t1" and carries no branch option ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 5b8f333f0..4718d92a1 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -2283,7 +2283,7 @@ TEST_P(DedicatedVectorTableTest, TestDedicatedVectorStore) { ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(invalid_builder_load_options, std::move(invalid_context), invalid_builder_options) .status(), - "requires row-tracking.enabled and data-evolution.enabled"); + "Cannot change immutable option 'data-evolution.enabled' dynamically"); } TEST_P(DedicatedVectorTableTest, TestIndependentVectorRolling) { diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 019aa567f..61a10e071 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -47,6 +47,8 @@ #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/io/data_file_meta.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/deletion_file.h" #include "paimon/core/table/source/key_value_table_read.h" @@ -85,6 +87,24 @@ using ScanAndReadParam = std::tuple; class ScanAndReadInteTest : public testing::Test, public ::testing::WithParamInterface { public: + // Reuse the fixture's records with a schema declaring the engine under test. Merge engines + // are immutable runtime options, so both operators must load that schema from the start. + Result LoadWithMergeEngine(const std::string& path, + const std::string& merge_engine) const { + SchemaManager manager(std::make_shared(), path); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + manager.Latest()); + if (!latest) { + return Status::Invalid("test fixture has no schema"); + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr schema, + latest.value()->MergeOptions({{Options::MERGE_ENGINE, merge_engine}})); + TableLoadOptions load{path}; + PAIMON_ASSIGN_OR_RAISE(load.table_schema_json, schema->GetJsonSchema()); + return load; + } + void CheckStreamScanResult( const std::unique_ptr& table_scan, const std::unique_ptr& table_read, const std::vector>& expected_snapshot_ids, @@ -2179,11 +2199,11 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + ASSERT_OK_AND_ASSIGN(TableLoadOptions scan_context_builder_load_options, + LoadWithMergeEngine(table_path, "aggregation")); std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; - scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); @@ -2193,11 +2213,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho TableScan::Create(scan_context_builder_load_options, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadOptions read_context_builder_load_options = scan_context_builder_load_options; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); - read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); @@ -2230,11 +2249,11 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + ASSERT_OK_AND_ASSIGN(TableLoadOptions scan_context_builder_load_options, + LoadWithMergeEngine(table_path, "aggregation")); std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; - scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; scan_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); @@ -2244,11 +2263,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh TableScan::Create(scan_context_builder_load_options, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadOptions read_context_builder_load_options = scan_context_builder_load_options; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); - read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; read_context_builder_options["fields.f3.aggregate-function"] = "rbm32"; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); @@ -2268,11 +2286,11 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + ASSERT_OK_AND_ASSIGN(TableLoadOptions scan_context_builder_load_options, + LoadWithMergeEngine(table_path, "aggregation")); std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; - scan_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; scan_context_builder_options["fields.f3.aggregate-function"] = "sum"; scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); @@ -2286,11 +2304,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica TableScan::Create(scan_context_builder_load_options, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadOptions read_context_builder_load_options = scan_context_builder_load_options; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); - read_context_builder_options[Options::MERGE_ENGINE] = "aggregation"; read_context_builder_options["fields.f3.aggregate-function"] = "sum"; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); @@ -2323,11 +2340,11 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + ASSERT_OK_AND_ASSIGN(TableLoadOptions scan_context_builder_load_options, + LoadWithMergeEngine(table_path, "partial-update")); std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; - scan_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; scan_context_builder_options[Options::IGNORE_DELETE] = "true"; scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); @@ -2341,11 +2358,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre TableScan::Create(scan_context_builder_load_options, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadOptions read_context_builder_load_options = scan_context_builder_load_options; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); - read_context_builder_options[Options::MERGE_ENGINE] = "partial-update"; read_context_builder_options[Options::IGNORE_DELETE] = "true"; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 1a2a02200..d01a79480 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -1896,7 +1896,7 @@ TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { FinishScanContext(context_builder, context_builder_options)); ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), context_builder_options), - "do not support pk table bucket=-1 in scan process"); + "Cannot change bucket from or to -1 dynamically"); } TEST_P(ScanInteTest, TestReadWithNoSnapshot) { From 6c20afa111ed342f1adb3caacb59cef063c79f63 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 13 Nov 2026 21:52:30 +0800 Subject: [PATCH 3/6] fix(context): default initialize table loading options --- include/paimon/catalog/table_load_options.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/paimon/catalog/table_load_options.h b/include/paimon/catalog/table_load_options.h index 3203f41c9..5bf12fb4c 100644 --- a/include/paimon/catalog/table_load_options.h +++ b/include/paimon/catalog/table_load_options.h @@ -34,18 +34,18 @@ class FileSystem; /// A catalog requires an identifier; a path is required for file-system loading. struct PAIMON_EXPORT TableLoadOptions { /// Optional for catalog loads; otherwise the native table or system-table path. - std::string path; + std::string path = {}; /// Explicit storage binding, taking precedence over the catalog's per-table file system. - std::shared_ptr file_system; + std::shared_ptr file_system = nullptr; /// URI scheme routing; cannot be specified together with file_system. - std::map fs_scheme_to_identifier_map; + std::map fs_scheme_to_identifier_map = {}; /// Serialized Paimon table schema, independent of the reader's Arrow projection. /// Used for the main branch, including a fallback to main from another branch. - std::optional table_schema_json; - std::shared_ptr catalog; + std::optional table_schema_json = std::nullopt; + std::shared_ptr catalog = nullptr; /// Requires catalog. A named branch must agree with the dynamic branch option. - std::optional identifier; + std::optional identifier = std::nullopt; /// Expected catalog UUID, or the UUID for an external REST commit request. - std::optional table_id; + std::optional table_id = std::nullopt; }; } // namespace paimon From ac052baa1f87bac3f299f2608f456f53dc3b8327 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 13 Nov 2026 22:50:16 +0800 Subject: [PATCH 4/6] refactor(context): rename table load options to table load context --- benchmark/benchmark_suite.cpp | 18 +- docs/source/api/catalog.rst | 2 +- docs/source/api/format_table.rst | 2 +- docs/source/user_guide/catalog.rst | 49 +- docs/source/user_guide/format_table.rst | 4 +- docs/source/user_guide/global_index.rst | 14 +- .../user_guide/manifest_entry_cache.rst | 4 +- docs/source/user_guide/read.rst | 4 +- docs/source/user_guide/realtime.rst | 12 +- docs/source/user_guide/write.rst | 2 +- examples/clean_demo.cpp | 10 +- examples/read_write_demo.cpp | 2 +- ...le_load_options.h => table_load_context.h} | 4 +- include/paimon/file_store_commit.h | 12 +- include/paimon/file_store_write.h | 6 +- include/paimon/table/source/table_read.h | 6 +- include/paimon/table/source/table_scan.h | 6 +- .../append_compact_coordinator_test.cpp | 10 +- .../core/catalog/file_system_catalog_test.cpp | 10 +- .../global_index/global_index_write_task.cpp | 8 +- ...eted_primary_key_index_maintainer_test.cpp | 30 +- ...erge_tree_compact_manager_factory_test.cpp | 6 +- .../merge_tree_compact_rewriter_test.cpp | 6 +- .../append_only_file_store_scan_test.cpp | 14 +- .../append_only_file_store_write_test.cpp | 44 +- .../core/operation/commit_context_test.cpp | 2 +- .../data_evolution_split_read_test.cpp | 64 +- .../operation/file_store_commit_impl_test.cpp | 428 ++++++------ .../core/operation/file_store_commit_test.cpp | 24 +- .../core/operation/file_store_write_test.cpp | 88 +-- .../operation/internal_read_context_test.cpp | 196 +++--- .../key_value_file_store_write_test.cpp | 74 +- .../operation/merge_file_split_read_test.cpp | 142 ++-- .../precise_file_index_read_test.cpp | 130 ++-- .../operation/raw_file_split_read_test.cpp | 150 ++-- .../core/operation/read_context_test.cpp | 2 +- .../core/operation/scan_context_test.cpp | 2 +- .../core/operation/write_context_test.cpp | 2 +- src/paimon/core/snapshot_file_scan_test.cpp | 10 +- .../core/table/format/format_table_test.cpp | 144 ++-- .../core/table/sink/commit_message_test.cpp | 8 +- .../table/source/append_count_reader_test.cpp | 10 +- .../table/source/filtered_row_id_reader.cpp | 8 +- .../table/source/pk_count_reader_test.cpp | 18 +- .../source/scan_exception_safety_test.cpp | 10 +- .../core/table/source/table_read_test.cpp | 116 ++-- .../core/table/source/table_scan_test.cpp | 26 +- .../core/table/system/system_table_test.cpp | 2 +- src/paimon/core/table/table_factory.cpp | 36 +- src/paimon/core/table/table_factory.h | 4 +- src/paimon/core/table/table_test.cpp | 322 ++++----- src/paimon/core/utils/blob_view_lookup.cpp | 18 +- src/paimon/rest/rest_catalog_test.cpp | 58 +- src/paimon/testing/utils/test_helper.h | 22 +- test/inte/append_compaction_inte_test.cpp | 10 +- test/inte/blob_table_inte_test.cpp | 26 +- test/inte/clean_inte_test.cpp | 70 +- test/inte/data_evolution_table_test.cpp | 74 +- test/inte/global_index_test.cpp | 38 +- test/inte/nested_column_pruning_inte_test.cpp | 46 +- test/inte/paimon_read_compat_inte_test.cpp | 10 +- test/inte/pk_compaction_inte_test.cpp | 62 +- .../primary_key_sorted_index_inte_test.cpp | 18 +- test/inte/read_inte_test.cpp | 212 +++--- test/inte/realtime_write_inte_test.cpp | 210 +++--- test/inte/scan_and_read_inte_test.cpp | 654 +++++++++--------- test/inte/scan_inte_test.cpp | 208 +++--- test/inte/write_and_read_inte_test.cpp | 118 ++-- .../write_and_read_inte_with_index_test.cpp | 30 +- test/inte/write_inte_test.cpp | 166 ++--- 70 files changed, 2177 insertions(+), 2176 deletions(-) rename include/paimon/catalog/{table_load_options.h => table_load_context.h} (94%) diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp index 307ee6abc..fc9d58944 100644 --- a/benchmark/benchmark_suite.cpp +++ b/benchmark/benchmark_suite.cpp @@ -37,7 +37,7 @@ #include "benchmark/cli_option_parsing.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/testing/utils/testharness.h" #if __has_include("parquet/arrow/reader.h") @@ -435,7 +435,7 @@ Status WriteSourceDataToWriter(paimon::FileStoreWrite* writer, const SourceDataS Status WriteAndCommit(const std::string& table_path, const std::map& options, const SourceDataSpec& source_spec) { - paimon::TableLoadOptions write_builder_load_options{table_path}; + paimon::TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; paimon::WriteContextBuilder write_builder("benchmark-writer"); write_builder_options = options; @@ -443,7 +443,7 @@ Status WriteAndCommit(const std::string& table_path, AddContext(write_builder.Finish(), "create write context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr writer, - AddContext(paimon::FileStoreWrite::Create(write_builder_load_options, std::move(write_ctx), + AddContext(paimon::FileStoreWrite::Create(write_builder_load_context, std::move(write_ctx), write_builder_options), "create file store writer")); @@ -451,7 +451,7 @@ Status WriteAndCommit(const std::string& table_path, PAIMON_ASSIGN_OR_RAISE(std::vector> messages, AddContext(writer->PrepareCommit(), "prepare commit")); - paimon::TableLoadOptions commit_builder_load_options{table_path}; + paimon::TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; paimon::CommitContextBuilder commit_builder("benchmark-writer"); commit_builder_options = options; @@ -459,7 +459,7 @@ Status WriteAndCommit(const std::string& table_path, AddContext(commit_builder.Finish(), "create commit context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr committer, - AddContext(paimon::FileStoreCommit::Create(commit_builder_load_options, + AddContext(paimon::FileStoreCommit::Create(commit_builder_load_context, std::move(commit_ctx), commit_builder_options), "create committer")); PAIMON_RETURN_NOT_OK(AddContext(committer->Commit(messages), "commit write")); @@ -571,7 +571,7 @@ Result GetOrCreateSharedReadTable(const std::string Result ReadRows(const std::string& table_path, const std::map& options, int32_t prefetch_parallel_num) { - paimon::TableLoadOptions scan_builder_load_options{table_path}; + paimon::TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; paimon::ScanContextBuilder scan_builder; scan_builder_options = options; @@ -579,13 +579,13 @@ Result ReadRows(const std::string& table_path, AddContext(scan_builder.Finish(), "create scan context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scanner, - AddContext(paimon::TableScan::Create(scan_builder_load_options, std::move(scan_ctx), + AddContext(paimon::TableScan::Create(scan_builder_load_context, std::move(scan_ctx), scan_builder_options), "create scanner")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, AddContext(scanner->CreatePlan(), "create plan")); - paimon::TableLoadOptions read_builder_load_options{table_path}; + paimon::TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; paimon::ReadContextBuilder read_builder; constexpr int32_t kPrefetchBatchCount = 600; @@ -599,7 +599,7 @@ Result ReadRows(const std::string& table_path, AddContext(read_builder.Finish(), "create read context")); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr reader, - AddContext(paimon::TableRead::Create(read_builder_load_options, std::move(read_ctx), + AddContext(paimon::TableRead::Create(read_builder_load_context, std::move(read_ctx), read_builder_options), "create table reader")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, diff --git a/docs/source/api/catalog.rst b/docs/source/api/catalog.rst index df19a17c6..4afe91c2d 100644 --- a/docs/source/api/catalog.rst +++ b/docs/source/api/catalog.rst @@ -37,7 +37,7 @@ Interface .. doxygenclass:: paimon::Table :members: -.. doxygenstruct:: paimon::TableLoadOptions +.. doxygenstruct:: paimon::TableLoadContext :members: .. doxygenstruct:: paimon::CatalogOptions diff --git a/docs/source/api/format_table.rst b/docs/source/api/format_table.rst index 7464205f7..8eeb88078 100644 --- a/docs/source/api/format_table.rst +++ b/docs/source/api/format_table.rst @@ -35,7 +35,7 @@ and :cpp:class:`paimon::FileStoreCommit`. See :doc:`../user_guide/format_table`. Contexts ======== -Static entry points load the table through :cpp:struct:`paimon::TableLoadOptions`. +Static entry points load the table through :cpp:struct:`paimon::TableLoadContext`. Operation Contexts provide execution settings; dynamic options are a separate argument. A caller holding a loaded ``FormatTable`` creates operators directly with its ``NewScan()``, ``NewRead()``, ``NewWrite()`` or ``NewCommit()`` methods. diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 9a3aaabd6..f40652c69 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -156,23 +156,24 @@ bound schema without metadata I/O; reload through the catalog to refresh it. Branch or type options that contradict a loaded managed table are rejected. See the :ref:`Table API `. -The static factories instead take ``TableLoadOptions``, an operation Context and -an optional dynamic options map. Loading parameters are independent of the -Context; finishing a Context performs no table metadata I/O: +The static factories instead take ``TableLoadContext``, an operation Context and +an optional dynamic options map. ``TableLoadContext`` holds the table identity, +storage binding and supplied metadata. Operation Contexts hold execution settings; +finishing an operation Context performs no table metadata I/O: .. code-block:: cpp - paimon::TableLoadOptions load_options; - load_options.catalog = shared_catalog; - load_options.identifier.emplace("db", "tbl"); + paimon::TableLoadContext load_context; + load_context.catalog = shared_catalog; + load_context.identifier.emplace("db", "tbl"); // Optional expected UUID, validated against the catalog during loading. - load_options.table_id = table->CatalogUuid(); + load_context.table_id = table->CatalogUuid(); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, paimon::CommitContextBuilder("commit-user").Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - paimon::FileStoreCommit::Create(load_options, std::move(context))); + paimon::FileStoreCommit::Create(load_context, std::move(context))); -For file-system loading, set ``TableLoadOptions::path``. Optional fields specify +For file-system loading, set ``TableLoadContext::path``. Optional fields specify an explicit file system, URI scheme routing or a serialized Paimon table schema. The serialized table schema is separate from the Arrow projection in ``ReadContextBuilder::SetReadSchema()``. Set the branch through the identifier @@ -191,11 +192,11 @@ under the table path. Recreating the writer restores offsets from the catalog's current snapshot. Table loading uses ``Catalog::GetTableFileSystem`` for per-table credentials. -An explicit ``TableLoadOptions::file_system`` or +An explicit ``TableLoadContext::file_system`` or ``fs_scheme_to_identifier_map`` controls file-system resolution instead. Loaded format tables create operators directly with their ``New*()`` methods. -A file system passed in ``TableLoadOptions::file_system`` is used exactly as it is: neither the +A file system passed in ``TableLoadContext::file_system`` is used exactly as it is: neither the built-in file systems nor the table's own credentials are involved, so a custom one has to authenticate its own accesses. Draw credentials that expire from a ``CredentialProvider`` you build with ``CredentialProviderFactory``: hold the @@ -209,9 +210,9 @@ returns credentials that are still valid and reloads them before they expire. "oss://bucket/tbl", options)); // MyFileSystem merges provider->GetCredentials() over its own options per access. auto my_fs = std::make_shared(provider); - load_options.catalog = shared_catalog; - load_options.identifier.emplace("db", "tbl"); - load_options.file_system = my_fs; + load_context.catalog = shared_catalog; + load_context.identifier.emplace("db", "tbl"); + load_context.file_system = my_fs; The snapshot being committed carries a uuid generated on the client, and the commit names the snapshot it is based on by that snapshot's uuid, so the server @@ -290,7 +291,7 @@ file-system lookup; other catalog errors propagate to the caller. Load the table before preparing changes so its retained ``CatalogUuid()`` lets the server reject commits to a dropped and recreated table. Static commit -loading accepts the expected UUID through ``TableLoadOptions::table_id``. +loading accepts the expected UUID through ``TableLoadContext::table_id``. ``Table::Uuid()`` can fall back to the table name and must not supply this id. An unset table id is sent as null and subject to server validation. Each commit attempt to the main branch reloads the catalog's current schema id for the new @@ -358,7 +359,7 @@ metadata I/O. The loaded table retains its schema, location, identifier, catalog services and per-table file system. That file system signs metadata and data accesses with the table's credentials and refreshes them as they expire. -Static factories take ``TableLoadOptions`` separately. Its explicit ``file_system`` +Static factories take ``TableLoadContext`` separately. Its explicit ``file_system`` takes precedence over the catalog's file system. Without one, the catalog supplies its per-table credentials, or its catalog-level file system when it issues none. Use ``NewScan`` and ``NewRead`` on a loaded format table too; its storage binding @@ -380,7 +381,7 @@ any of these sets it: branch read, which ``TableScan::Create`` and ``TableRead::Create`` load as Java loads the table of that branch. -A schema handed in through ``TableLoadOptions::table_schema_json`` for the main branch does not decide it +A schema handed in through ``TableLoadContext::table_schema_json`` for the main branch does not decide it alone: unless the catalog already enabled the option, the latest schema under the table path is checked too, so a cache taken before the option was turned on still applies the rules. @@ -551,7 +552,7 @@ These examples use ``shared_catalog`` from `Reading through the catalog`_. using namespace paimon; // Plan with the columns and the filter the read uses. - TableLoadOptions scan_load{table_path}; + TableLoadContext scan_load{table_path}; scan_load.catalog = shared_catalog; scan_load.identifier.emplace("db", "tbl"); ScanContextBuilder scan_builder; @@ -563,7 +564,7 @@ These examples use ``shared_catalog`` from `Reading through the catalog`_. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); // A worker without the catalog tells the read that the table enables query auth. - TableLoadOptions read_load{table_path}; + TableLoadContext read_load{table_path}; ReadContextBuilder read_builder; read_builder.SetReadFieldNames({"name", "phone"}) .SetPredicate(predicate) @@ -577,13 +578,13 @@ These examples use ``shared_catalog`` from `Reading through the catalog`_. The read does not take the branch from the splits or from ``query-auth.enabled``: a worker reading a branch names it itself, as the scan did. As for a scan, a branch named by the dynamic ``branch`` option is -carried into the identifier supplied in ``TableLoadOptions``, so the catalog +carried into the identifier supplied in ``TableLoadContext``, so the catalog hands out the credentials of that branch. .. code-block:: cpp // Plan branch "dev"; Identifier("db", "tbl", "dev") names it as well. - TableLoadOptions dev_load{table_path}; + TableLoadContext dev_load{table_path}; dev_load.catalog = shared_catalog; dev_load.identifier.emplace("db", "tbl", "dev"); ScanContextBuilder dev_scan_builder; @@ -595,7 +596,7 @@ hands out the credentials of that branch. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr dev_plan, dev_scan->CreatePlan()); // The worker reads the same branch. - TableLoadOptions dev_read_load{table_path}; + TableLoadContext dev_read_load{table_path}; ReadContextBuilder dev_read_builder; dev_read_builder.SetReadFieldNames({"name", "phone"}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr dev_read_context, @@ -621,7 +622,7 @@ This library creates no branch: one has to exist, with its schema published unde Naming the branch ^^^^^^^^^^^^^^^^^ -A branch is addressed by ``TableLoadOptions::identifier`` or the loaded table identifier, which +A branch is addressed by ``TableLoadContext::identifier`` or the loaded table identifier, which ``paimon::Identifier("db", "tbl", "dev")`` builds as ``tbl$branch_dev``, so that the catalog answers for the branch: the snapshot it loads is the branch's, and the commit request is addressed to that branch by that identifier rather than by @@ -680,7 +681,7 @@ from a store Paimon has no client for — reach a file system through a ``REGISTER_PAIMON_FACTORY``. Build a provider with ``CredentialProviderFactory::Get``, wrap it in a ``FileSystem`` of your own that calls ``CredentialProvider::GetCredentials`` as it signs each access, and pass -that file system in through ``Catalog::Create`` or ``TableLoadOptions::file_system``. +that file system in through ``Catalog::Create`` or ``TableLoadContext::file_system``. A file system passed this way is used as-is, so the credentials are reloaded as they expire without it being rebuilt. diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 7caadce2d..9d14200cd 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -49,7 +49,7 @@ carry a format-table handle, its identity. Conflicting contexts are rejected before creating an operator, including for overwrite commits. Static ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` -and ``FileStoreCommit::Create`` load tables through ``TableLoadOptions``. They +and ``FileStoreCommit::Create`` load tables through ``TableLoadContext``. They read the schema from the table path or ``table_schema_json`` and dispatch by its type. Operation Contexts contain execution settings; dynamic options are a separate final argument. @@ -126,7 +126,7 @@ A setting the format path cannot act on is refused by name rather than quietly d * ``WriteContextBuilder::WithWriteId()``, which prefixes a postpone-bucket writer's files so one compaction reader can put them back in order; a format table has no buckets. * ``CommitContextBuilder::IgnoreEmptyCommit(false)``, ``UseRESTCatalogCommit(true)``, - ``TableLoadOptions::table_id`` and ``AppendCommitCheckConflict(true)``. Keeping an empty + ``TableLoadContext::table_id`` and ``AppendCommitCheckConflict(true)``. Keeping an empty commit means writing a snapshot that adds no files, a REST commit sends that snapshot to a catalog, a table id names the table in that request, and the conflict check reads the manifests of concurrent commits - none of which exist here. Each is refused only when set away from its default, so an diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 1cb7e8c2f..3d9663879 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -223,7 +223,7 @@ to both the index scan and the data scan: reader->VisitFullTextSearch(std::make_shared( "f0", R"({"match":{"query":"paimon lake"}})", /*limit=*/10))); - paimon::TableLoadOptions load_options{table_path}; + paimon::TableLoadContext load_context{table_path}; paimon::ScanContextBuilder scan_builder; const std::map scan_options = { {paimon::Options::SCAN_MODE, "from-snapshot"}, @@ -231,7 +231,7 @@ to both the index scan and the data scan: scan_builder.SetGlobalIndexResult(result); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(load_options, std::move(context), scan_options)); + paimon::TableScan::Create(load_context, std::move(context), scan_options)); **Configuration:** @@ -318,13 +318,13 @@ them. A search whose ``full-text-index.search-mode``, or else ``global-index.sea search_builder.ExecuteLocal()); // Read the rows found, with their scores. - paimon::TableLoadOptions load_options{table_path}; + paimon::TableLoadContext load_context{table_path}; paimon::ScanContextBuilder scan_builder; scan_builder.SetGlobalIndexResult(result); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(load_options, std::move(scan_context))); + paimon::TableScan::Create(load_context, std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_plan, scan->CreatePlan()); paimon::ReadContextBuilder read_builder; @@ -332,7 +332,7 @@ them. A search whose ``full-text-index.search-mode``, or else ``global-index.sea PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - paimon::TableRead::Create(load_options, std::move(read_context))); + paimon::TableRead::Create(load_context, std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(data_plan->Splits())); @@ -422,7 +422,7 @@ outlive the snapshot. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, search_read->Read(*plan)); - paimon::TableLoadOptions load_options{table_path}; + paimon::TableLoadContext load_context{table_path}; paimon::ScanContextBuilder scan_builder; scan_builder.SetGlobalIndexResult(result); std::map scan_options; @@ -432,7 +432,7 @@ outlive the snapshot. } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(load_options, std::move(context), scan_options)); + paimon::TableScan::Create(load_context, std::move(context), scan_options)); ``ExecuteLocal()`` plans and searches in one process. To plan in one process and search in others, plan with ``NewFullTextScan()``, ship the splits with ``FullTextSearchSplit::Serialize()`` and diff --git a/docs/source/user_guide/manifest_entry_cache.rst b/docs/source/user_guide/manifest_entry_cache.rst index ab55b230d..c1fc1ab9a 100644 --- a/docs/source/user_guide/manifest_entry_cache.rst +++ b/docs/source/user_guide/manifest_entry_cache.rst @@ -78,14 +78,14 @@ under .. code-block:: cpp auto cache = std::make_shared(128 * 1024 * 1024); - TableLoadOptions load_options{table_path}; + TableLoadContext load_context{table_path}; ScanContextBuilder context_builder; PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scan_context, context_builder.WithCache(cache).Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr scan, - TableScan::Create(load_options, std::move(scan_context), + TableScan::Create(load_context, std::move(scan_context), {{Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"}})); .. note:: diff --git a/docs/source/user_guide/read.rst b/docs/source/user_guide/read.rst index 76543e074..5566d2981 100644 --- a/docs/source/user_guide/read.rst +++ b/docs/source/user_guide/read.rst @@ -49,11 +49,11 @@ In a function returning ``Status`` or ``Result``, use the ordinary scan API: .. code-block:: cpp - TableLoadOptions load_options{table_path}; + TableLoadContext load_context{table_path}; ScanContextBuilder builder; builder.WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, TableScan::Create(load_options, std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, TableScan::Create(load_context, std::move(context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); * The pool controls only allocations routed through it, not all planning memory diff --git a/docs/source/user_guide/realtime.rst b/docs/source/user_guide/realtime.rst index 74ee104b0..bd21c8991 100644 --- a/docs/source/user_guide/realtime.rst +++ b/docs/source/user_guide/realtime.rst @@ -291,14 +291,14 @@ and an input ``batch`` built as described above. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime, RealtimeContext::Create()); - TableLoadOptions load_options{table_path}; + TableLoadContext load_context{table_path}; WriteContextBuilder write_builder(commit_user); write_builder.WithStreamingMode(true) .WithRealtimeContext(realtime); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - FileStoreWrite::Create(load_options, std::move(write_context), options)); + FileStoreWrite::Create(load_context, std::move(write_context), options)); PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch))); To query before preparing or committing: @@ -310,7 +310,7 @@ To query before preparing or committing: PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(load_options, std::move(scan_context), options)); + TableScan::Create(load_context, std::move(scan_context), options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); ReadContextBuilder read_builder; @@ -318,7 +318,7 @@ To query before preparing or committing: PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(load_options, std::move(read_context), options)); + TableRead::Create(load_context, std::move(read_context), options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); // Consume NextBatch() or NextBatchWithBitmap(), then Close() the reader. @@ -340,7 +340,7 @@ Checkpoint with progress-aware APIs: PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - FileStoreCommit::Create(load_options, std::move(commit_context), options)); + FileStoreCommit::Create(load_context, std::move(commit_context), options)); PAIMON_ASSIGN_OR_RAISE( int64_t snapshot_id, commit->CommitWithProgress(progress, checkpoint_id, /*watermark=*/std::nullopt)); @@ -374,7 +374,7 @@ coordinating checkpoints. Snapshot conflicts are retried internally against the latest snapshot; commit identifiers must follow the streaming commit protocol. With a version-managed catalog, configure the writer and committer from the same catalog-loaded Table, or use the same catalog and identifier in -``TableLoadOptions``. +``TableLoadContext``. Read Views and Consistency -------------------------- diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst index 9ac324226..66a0e5997 100644 --- a/docs/source/user_guide/write.rst +++ b/docs/source/user_guide/write.rst @@ -301,7 +301,7 @@ Operational Flow passes the objects to ``FileStoreCommit`` for either an Append or PK table. For a table of a catalog which owns its versions, such as a REST catalog, it passes that catalog and the table identifier to - a shared catalog-loaded Table or matching ``TableLoadOptions``. + a shared catalog-loaded Table or matching ``TableLoadContext``. The writer loads the latest snapshot from the catalog, and the current schema from it as well when it writes to the main branch; a write aimed at another branch reads the schema published under ``branch/branch-`` instead. On diff --git a/examples/clean_demo.cpp b/examples/clean_demo.cpp index c5e4e1ecb..b11ea727a 100644 --- a/examples/clean_demo.cpp +++ b/examples/clean_demo.cpp @@ -26,7 +26,7 @@ #include "arrow/ipc/api.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/orphan_files_cleaner.h" namespace fs = std::filesystem; @@ -48,14 +48,14 @@ Status CleanOrphanFiles(const std::string& table_path, int64_t older_than_ms) { Status DropPartition(const std::string& table_path, const std::vector>& partitions) { - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(/*commit_user=*/"commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr committer, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); PAIMON_RETURN_NOT_OK(committer->DropPartition(partitions, /*commit_identifier=*/10)); @@ -63,7 +63,7 @@ Status DropPartition(const std::string& table_path, } Status ExpireSnapshot(const std::string& table_path) { - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(/*commit_user=*/"commit_user_1"); std::map commit_options = { @@ -76,7 +76,7 @@ Status ExpireSnapshot(const std::string& table_path) { commit_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr committer, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); PAIMON_RETURN_NOT_OK(committer->Expire()); diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index 51ddc027d..1bbe67a02 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -27,7 +27,7 @@ #include "paimon/api.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/table.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" arrow::Result> PrepareData(const arrow::FieldVector& fields) { arrow::StringBuilder f0_builder; diff --git a/include/paimon/catalog/table_load_options.h b/include/paimon/catalog/table_load_context.h similarity index 94% rename from include/paimon/catalog/table_load_options.h rename to include/paimon/catalog/table_load_context.h index 5bf12fb4c..8737bdee5 100644 --- a/include/paimon/catalog/table_load_options.h +++ b/include/paimon/catalog/table_load_context.h @@ -30,9 +30,9 @@ namespace paimon { class Catalog; class FileSystem; -/// Parameters used only while loading a table for a static operator factory. +/// Context used only while loading a table for a static operator factory. /// A catalog requires an identifier; a path is required for file-system loading. -struct PAIMON_EXPORT TableLoadOptions { +struct PAIMON_EXPORT TableLoadContext { /// Optional for catalog loads; otherwise the native table or system-table path. std::string path = {}; /// Explicit storage binding, taking precedence over the catalog's per-table file system. diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index 0a1b505b4..2af9209ba 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -26,7 +26,7 @@ #include #include -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" @@ -47,19 +47,19 @@ class CommitMessage; /// dropping partitions, and retrieving commit metrics. /// /// @note Direct file-system commits support append-only and primary-key tables on non-object-store -/// paths. `TableLoadOptions::catalog` also supports object stores when the catalog +/// paths. `TableLoadContext::catalog` also supports object stores when the catalog /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: /// Create an instance of `FileStoreCommit`. /// - /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param load_context Table identity, metadata and file-system loading parameters. /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `CommitContext` used for commit operations. /// /// @return A Result containing a unique pointer to the `FileStoreCommit` instance. static Result> Create( - const TableLoadOptions& load_options, std::unique_ptr context, + const TableLoadContext& load_context, std::unique_ptr context, const std::map& dynamic_options = {}); virtual ~FileStoreCommit() = default; @@ -98,10 +98,10 @@ class PAIMON_EXPORT FileStoreCommit { /// An error is terminal for the writer state which produced `realtime_commits`. The caller must /// discard its `RealtimeContext` and `FileStoreWrite`, load the current latest snapshot's /// durable offsets, recreate both objects, and replay input from those exclusive offsets. - /// Conflicts from `TableLoadOptions::catalog` are retried internally. With + /// Conflicts from `TableLoadContext::catalog` are retried internally. With /// `UseRESTCatalogCommit()`, the caller sends the request and handles server conflicts. /// Concurrent rollback or partition deletion must be fenced by the upstream coordinator. - /// Load writers through the same catalog using TableLoadOptions or Catalog::GetTable(). + /// Load writers through the same catalog using TableLoadContext or Catalog::GetTable(). /// /// In deduplicate mode each commit attempt resolves logical deleted offsets against the /// latest snapshot and merges them with its deletion vectors. Keep the complete progress diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 4205df0e7..686e5135a 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -25,7 +25,7 @@ #include #include -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_message.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -49,13 +49,13 @@ class PAIMON_EXPORT FileStoreWrite { public: /// Create an instance of `FileStoreWrite`. /// - /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param load_context Table identity, metadata and file-system loading parameters. /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `WriteContext` used for write operations. /// /// @return A Result containing a unique pointer to the `FileStoreWrite` instance. static Result> Create( - const TableLoadOptions& load_options, std::unique_ptr context, + const TableLoadContext& load_context, std::unique_ptr context, const std::map& dynamic_options = {}); virtual ~FileStoreWrite() = default; diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index 46485459a..b093aa11b 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -22,7 +22,7 @@ #include #include -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/executor.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" @@ -45,12 +45,12 @@ class PAIMON_EXPORT TableRead { /// Create an instance of `TableRead`. /// - /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param load_context Table identity, metadata and file-system loading parameters. /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. static Result> Create( - const TableLoadOptions& load_options, std::unique_ptr context, + const TableLoadContext& load_context, std::unique_ptr context, const std::map& dynamic_options = {}); /// Creates a `BatchReader` instance for reading data. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index e02a059f9..aa2e4fe42 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -24,7 +24,7 @@ #include #include -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/result.h" #include "paimon/table/source/plan.h" #include "paimon/table/source/scan_metrics.h" @@ -39,12 +39,12 @@ class PAIMON_EXPORT TableScan { public: /// Create an instance of `TableScan`. /// - /// @param load_options Table identity, metadata and file-system loading parameters. + /// @param load_context Table identity, metadata and file-system loading parameters. /// @param dynamic_options Overrides merged on top of the table options. /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. static Result> Create( - const TableLoadOptions& load_options, std::unique_ptr context, + const TableLoadContext& load_context, std::unique_ptr context, const std::map& dynamic_options = {}); virtual ~TableScan() = default; diff --git a/src/paimon/core/append/append_compact_coordinator_test.cpp b/src/paimon/core/append/append_compact_coordinator_test.cpp index df11f129c..cc92025a9 100644 --- a/src/paimon/core/append/append_compact_coordinator_test.cpp +++ b/src/paimon/core/append/append_compact_coordinator_test.cpp @@ -29,7 +29,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/path_util.h" @@ -80,14 +80,14 @@ class AppendCompactCoordinatorTest : public ::testing::Test { const std::map& partition, int32_t bucket, const std::shared_ptr& write_array, int64_t commit_identifier) { - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); @@ -102,14 +102,14 @@ class AppendCompactCoordinatorTest : public ::testing::Test { Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_store_commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); return file_store_commit->Commit(commit_msgs); } diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index bb6f10eca..c4ac842df 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -26,7 +26,7 @@ #include "arrow/c/helpers.h" #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/path_util.h" @@ -1488,16 +1488,16 @@ TEST(FileSystemCatalogTest, TestClaimingVersionManagementWithoutImplementingItIs ASSERT_NOK_WITH_MSG(CheckVersionManagementImplemented(claiming), "does not implement VersionManagedCatalog"); - TableLoadOptions claiming_builder_load_options{dir->Str()}; + TableLoadContext claiming_builder_load_context{dir->Str()}; std::map claiming_builder_options; CommitContextBuilder claiming_builder("commit_user_1"); claiming_builder_options = options; - claiming_builder_load_options.catalog = claiming; - claiming_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + claiming_builder_load_context.catalog = claiming; + claiming_builder_load_context.identifier.emplace(Identifier("db", "tbl")); ASSERT_OK_AND_ASSIGN(std::unique_ptr claiming_context, claiming_builder.Finish()); ASSERT_NOK_WITH_MSG( - FileStoreCommit::Create(claiming_builder_load_options, std::move(claiming_context), + FileStoreCommit::Create(claiming_builder_load_context, std::move(claiming_context), claiming_builder_options), "does not implement VersionManagedCatalog"); diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index ff3322780..db25ade23 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -26,7 +26,7 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "arrow/type.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/mem_utils.h" @@ -178,18 +178,18 @@ Result> CreateBatchReader( const std::string& table_path, const std::vector& read_field_names, const std::shared_ptr& indexed_split, const CoreOptions& core_options, const std::shared_ptr& pool) { - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = core_options.ToMap(); - read_context_builder_load_options.file_system = core_options.GetFileSystem(); + read_context_builder_load_context.file_system = core_options.GetFileSystem(); read_context_builder.EnablePrefetch(true).WithMemoryPool(pool).SetReadFieldNames( read_field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); return table_read->CreateReader(indexed_split); } diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index c2d0e3d7a..41c94cdb2 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -33,7 +33,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" @@ -321,7 +321,7 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam> CreateWriter(bool with_temp_directory = true, bool write_buffer_spillable = true) const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; WriteContextBuilder builder(kCommitUser); std::map writer_options = options_; @@ -332,7 +332,7 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParamStr()); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(builder_load_options, std::move(context), builder_options); + return FileStoreWrite::Create(builder_load_context, std::move(context), builder_options); } Result>> WriteAndPrepare( @@ -358,7 +358,7 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam>& messages, int64_t commit_identifier) const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder(kCommitUser); builder_options = options_; @@ -366,7 +366,7 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); return commit->Commit(messages, commit_identifier); } @@ -462,14 +462,14 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); - TableLoadOptions scan_builder_load_options{table_path_}; + TableLoadContext scan_builder_load_context{table_path_}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; @@ -484,7 +484,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_GT(indexed_split_count, 0); ASSERT_EQ(0, data_split_count); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; // The source build above keeps lazy decoding enabled. Disable it only for the final data-row @@ -495,7 +495,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); @@ -701,14 +701,14 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF std::shared_ptr value_predicate = PredicateBuilder::Equal( /*field_index=*/1, "value", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); - TableLoadOptions scan_builder_load_options{table_path_}; + TableLoadContext scan_builder_load_context{table_path_}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options[Options::GLOBAL_INDEX_ENABLED] = "true"; scan_builder.SetPredicate(value_predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); int32_t indexed_split_count = 0; @@ -723,7 +723,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ASSERT_EQ(0, indexed_split_count); ASSERT_GT(data_split_count, 0); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options["orc.read.enable-lazy-decoding"] = "false"; @@ -732,7 +732,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); @@ -785,7 +785,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, DeletesPayloadAfterLastDefinitionI PathUtil::JoinPath(table_path_, "schema/schema-1"), schema_json, /*overwrite=*/false)); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; WriteContextBuilder builder(kCommitUser); builder_options = evolved_options; @@ -793,7 +793,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, DeletesPayloadAfterLastDefinitionI ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr writer, - FileStoreWrite::Create(builder_load_options, std::move(context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK(writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN(std::vector> cleanup_messages, writer->PrepareCommit(/*wait_compaction=*/true, diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp index ea87d5e72..9521f3458 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_factory_test.cpp @@ -31,7 +31,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/path_util.h" @@ -116,7 +116,7 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { table_options, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("test"); if (!context_options.empty()) { @@ -128,7 +128,7 @@ class MergeTreeCompactManagerFactoryWriteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + return FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options); } diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp index 73c634ddc..722a09091 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp @@ -21,7 +21,7 @@ #include "arrow/api.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" #include "paimon/core/mergetree/compact/interval_partition.h" @@ -58,12 +58,12 @@ class MergeTreeCompactRewriterTest : public testing::Test { Result>> GenerateSortedRuns( const std::string& table_path, const std::shared_ptr& table_schema, int32_t bucket, const std::map& partition) const { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetBucketFilter(bucket).SetPartitionFilter({partition}); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index 2f549bafe..87d467aad 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -30,7 +30,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/serializer/binary_row_serializer.h" @@ -600,7 +600,7 @@ TEST(AppendOnlyFileStoreScanTest, TestReconstructPredicateWithNonCastedFields) { TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = "orc"; @@ -610,7 +610,7 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { auto pool = GetDefaultPool(); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -651,7 +651,7 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) { TimezoneGuard guard("Asia/Shanghai"); std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = "orc"; @@ -660,7 +660,7 @@ TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) { ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); ASSERT_TRUE(typed_table_scan); @@ -757,7 +757,7 @@ std::shared_ptr BuildScan(const std::string& table_path, const std::shared_ptr& predicate = nullptr, bool manifest_entry_lazy_decode_enabled = true, int32_t max_snapshots = 8) { - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = "orc"; @@ -776,7 +776,7 @@ std::shared_ptr BuildScan(const std::string& table_path, } EXPECT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); EXPECT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); auto typed_table_scan = dynamic_cast(table_scan.get()); EXPECT_TRUE(typed_table_scan); diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index b8c28e2cc..bec5e87c4 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -36,7 +36,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" @@ -112,14 +112,14 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder(commit_user_); builder_options = options; EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(builder_load_options, std::move(write_context), + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( @@ -133,14 +133,14 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { const std::map& options, const std::vector& write_schema, const std::string& json, int64_t commit_identifier) const { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder(commit_user_); builder_options = options; builder.WithWriteSchema(write_schema); EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(builder_load_options, std::move(write_context), + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( @@ -151,14 +151,14 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; CommitContextBuilder builder(commit_user_); builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(builder_load_options, + FileStoreCommit::Create(builder_load_context, std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -238,12 +238,12 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - TableLoadOptions builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map builder_options; WriteContextBuilder builder(commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(builder_load_options, std::move(write_context), + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); ASSERT_NOK_WITH_MSG(file_store_write->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress is only supported by a real-time writer"); @@ -261,13 +261,13 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder(commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); auto array = std::make_shared(); arrow::StringBuilder builder; @@ -309,7 +309,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) { ASSERT_OK_AND_ASSIGN(std::shared_ptr batch_realtime_context, RealtimeContext::Create()); - TableLoadOptions batch_builder_load_options{table_path}; + TableLoadContext batch_builder_load_context{table_path}; std::map batch_builder_options; WriteContextBuilder batch_builder(commit_user_); batch_builder_options = options; @@ -317,7 +317,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_context, batch_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr batch_writer, - FileStoreWrite::Create(batch_builder_load_options, std::move(batch_context), + FileStoreWrite::Create(batch_builder_load_context, std::move(batch_context), batch_builder_options)); ASSERT_NOK_WITH_MSG(batch_writer->PrepareCommitWithProgress(/*commit_identifier=*/0), "PrepareCommitWithProgress requires streaming mode"); @@ -326,7 +326,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder(commit_user_); builder_options = options; @@ -334,7 +334,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(realtime_schema, R"([ [10, 1, "a"], @@ -407,8 +407,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) } TEST_F(AppendOnlyFileStoreWriteTest, TestGetMaxSequenceNumberFromMultiPartition) { - TableLoadOptions builder_load_options; - builder_load_options.path = + TableLoadContext builder_load_context; + builder_load_context.path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table/"; std::map builder_options; @@ -418,7 +418,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestGetMaxSequenceNumberFromMultiPartition) ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); auto write = dynamic_cast(file_store_write.get()); auto pool = GetDefaultPool(); { @@ -478,7 +478,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFi std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder(commit_user_); builder_options = options; @@ -486,7 +486,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFi ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]]] ])"))); @@ -591,7 +591,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMultipleMapColumnsAdaptA CreateTable(dir->Str(), logical_schema, options); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder(commit_user_); builder_options = options; @@ -599,7 +599,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMultipleMapColumnsAdaptA ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([ [1, [["a", 1], ["b", 2]], [["c", 3], ["d", 4], ["e", 5], ["f", 6]]] ])"))); diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index 6eb8752b3..359a7eb5b 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -24,7 +24,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/defs.h" #include "paimon/executor.h" diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index 02dbb1ddd..28e8d539a 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -29,7 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/file_index/bitmap/bitmap_file_index.h" @@ -412,16 +412,16 @@ TEST_F(DataEvolutionSplitReadTest, TestBlobFallbackValidatesRangesBeforeOpeningF {Options::DATA_EVOLUTION_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr context, InternalReadContext::Create( - read_context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); DataEvolutionSplitRead split_read(/*path_factory=*/nullptr, context, GetDefaultPool(), /*executor=*/nullptr); auto blob = @@ -456,16 +456,16 @@ TEST_F(DataEvolutionSplitReadTest, TestEmptyBlobReaders) { {Options::DATA_EVOLUTION_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr context, InternalReadContext::Create( - read_context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, FileStorePathFactory::Create(dir->Str(), schema, {}, "default", "parquet", "data-", @@ -1121,18 +1121,18 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { std::vector{"_ROW_ID", "_SEQUENCE_NUMBER"}}) { auto counting_fs = std::make_shared( PathUtil::JoinPath(bucket_path, main->file_name)); - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; ReadContextBuilder builder; - builder_load_options.file_system = counting_fs; + builder_load_context.file_system = counting_fs; builder.EnablePrefetch(false).SetReadFieldNames(projection); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr context, InternalReadContext::Create( - read_context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); auto split_read = std::make_unique(path_factory, context, GetDefaultPool(), executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, split_read->CreateReader(split)); @@ -1153,19 +1153,19 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { // A selection confined to the first VECTOR file must not include rows from the second. { - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; ReadContextBuilder builder; - builder_load_options.file_system = fs; + builder_load_context.file_system = fs; builder.EnablePrefetch(false).SetReadFieldNames( {"id", "embedding", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr context, InternalReadContext::Create( - read_context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); DataEvolutionSplitRead split_read(path_factory, context, GetDefaultPool(), executor); auto selected_split = std::make_shared(split, std::vector{Range(0, 1)}); @@ -1200,19 +1200,19 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { {vector1, legacy_main, vector0}); ASSERT_OK_AND_ASSIGN(std::shared_ptr legacy_split, legacy_split_builder.RawConvertible(false).Build()); - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; ReadContextBuilder builder; - builder_load_options.file_system = fs; + builder_load_context.file_system = fs; builder.EnablePrefetch(false).SetReadFieldNames( {"id", "embedding", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr context, InternalReadContext::Create( - read_context, evolved_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, evolved_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); DataEvolutionSplitRead split_read(path_factory, context, GetDefaultPool(), executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, split_read.CreateReader(legacy_split)); @@ -1428,16 +1428,16 @@ TEST_F(DataEvolutionSplitReadTest, TestFileIndexSelectionOnlyPushesDownNormalGro {Options::FILE_INDEX_READ_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr context, InternalReadContext::Create( - read_context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); DataEvolutionSplitRead split_read(/*path_factory=*/nullptr, context, pool, /*executor=*/nullptr); @@ -1608,16 +1608,16 @@ TEST_F(DataEvolutionSplitReadTest, TestGroupBitmapCombinesWithRowRangesAndDvInFi {Options::DATA_EVOLUTION_ENABLED, "true"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, TableSchema::Create(0, schema, {}, {}, options)); - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr context, InternalReadContext::Create( - read_context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); DataEvolutionSplitRead split_read(/*path_factory=*/nullptr, context, GetDefaultPool(), /*executor=*/nullptr); split_read.file_indexes_evaluated_ = true; diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index eebc7cb81..0b1bdd647 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -37,7 +37,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/data/binary_row.h" @@ -341,14 +341,14 @@ class FileStoreCommitImplTest : public testing::Test { } void CommitAsCompetitor(int32_t round, int64_t commit_identifier) { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder("competitor"); builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr competitor, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK(competitor->Commit(CommitMessagesOfRound(round), commit_identifier)); } @@ -364,22 +364,22 @@ class FileStoreCommitImplTest : public testing::Test { Result> CreateCatalogCommit( const CatalogCommitSpec& spec) const { - TableLoadOptions builder_load_options{spec.root_path.empty() ? table_path_ + TableLoadContext builder_load_context{spec.root_path.empty() ? table_path_ : spec.root_path}; std::map builder_options; CommitContextBuilder builder(spec.commit_user); builder_options[Options::FILE_SYSTEM] = "local"; - builder_load_options.catalog = spec.catalog; - builder_load_options.identifier.emplace(spec.identifier.value()); + builder_load_context.catalog = spec.catalog; + builder_load_context.identifier.emplace(spec.identifier.value()); builder.AppendCommitCheckConflict(spec.append_commit_check_conflict); if (spec.table_id) { - builder_load_options.table_id = spec.table_id.value(); + builder_load_context.table_id = spec.table_id.value(); } for (const auto& [key, value] : spec.options) { builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(builder_load_options, std::move(context), builder_options); + return FileStoreCommit::Create(builder_load_context, std::move(context), builder_options); } Result> CreateCatalogCommit( @@ -436,7 +436,7 @@ class FileStoreCommitImplTest : public testing::Test { void CheckExpireIsRefusedWithoutDeleting(const std::string& branch, const std::string& expected_msg) { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder("commit_user_1"); builder_options[Options::FILE_SYSTEM] = "local"; @@ -449,7 +449,7 @@ class FileStoreCommitImplTest : public testing::Test { .Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK(PrepareFirstCommitMessageFiles()); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK(commit->DropPartition({{{"f1", "10"}}}, /*commit_identifier=*/2)); @@ -638,7 +638,7 @@ class FileStoreCommitImplTest : public testing::Test { }; TEST_F(FileStoreCommitImplTest, TestCommit) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -646,7 +646,7 @@ TEST_F(FileStoreCommitImplTest, TestCommit) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = @@ -669,17 +669,17 @@ TEST_F(FileStoreCommitImplTest, TestCommit) { TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TimezoneGuard guard("Asia/Shanghai"); - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; - context_builder_load_options.table_id = "table-uuid"; + context_builder_load_context.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.UseRESTCatalogCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = CommitMessagesOfRound(1); @@ -730,13 +730,13 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { std::vector> msgs = CommitMessagesOfRound(1); - TableLoadOptions base_builder_load_options{table_path_}; + TableLoadContext base_builder_load_context{table_path_}; std::map base_builder_options; CommitContextBuilder base_builder("commit_user_1"); base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto base_commit, - FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + FileStoreCommit::Create(base_builder_load_context, std::move(base_context), base_builder_options)); ASSERT_OK(base_commit->Commit(msgs, 1)); @@ -745,15 +745,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitCarriesBaseSnapshotUuid) { ASSERT_TRUE(base_snapshot.Uuid().has_value()); const std::string base_uuid = base_snapshot.Uuid().value(); - TableLoadOptions rest_builder_load_options{table_path_}; + TableLoadContext rest_builder_load_context{table_path_}; std::map rest_builder_options; CommitContextBuilder rest_builder("commit_user_1"); rest_builder_options[Options::FILE_SYSTEM] = "local"; - rest_builder_load_options.table_id = "table-uuid"; + rest_builder_load_context.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, rest_builder.UseRESTCatalogCommit(true).Finish()); ASSERT_OK_AND_ASSIGN(auto rest_commit, - FileStoreCommit::Create(rest_builder_load_options, std::move(rest_context), + FileStoreCommit::Create(rest_builder_load_context, std::move(rest_context), rest_builder_options)); ASSERT_OK(rest_commit->Commit(msgs, 2)); @@ -814,14 +814,14 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitTakesTheSnapshot) { } TEST_F(FileStoreCommitImplTest, TestCommitToBranch) { - TableLoadOptions missing_builder_load_options{table_path_}; + TableLoadContext missing_builder_load_context{table_path_}; std::map missing_builder_options; CommitContextBuilder missing_builder("commit_user_1"); missing_builder_options[Options::FILE_SYSTEM] = "local"; missing_builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr missing_context, missing_builder.Finish()); ASSERT_NOK_WITH_MSG( - FileStoreCommit::Create(missing_builder_load_options, std::move(missing_context), + FileStoreCommit::Create(missing_builder_load_context, std::move(missing_context), missing_builder_options), "not found latest schema in branch dev"); @@ -829,7 +829,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitToBranch) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder("commit_user_1"); builder_options[Options::FILE_SYSTEM] = "local"; @@ -837,7 +837,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitToBranch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); std::string branch_snapshots = PathUtil::JoinPath(table_path_, "branch/branch-dev/snapshot"); @@ -863,17 +863,17 @@ TEST_F(FileStoreCommitImplTest, TestRequestOnlyCommitToBranch) { ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder("commit_user_1"); builder_options[Options::FILE_SYSTEM] = "local"; builder_options[Options::BRANCH] = "dev"; - builder_load_options.table_id = "table-uuid"; + builder_load_context.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.UseRESTCatalogCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); ASSERT_OK_AND_ASSIGN(std::string request_str, commit->GetLastCommitTableRequest()); @@ -984,14 +984,14 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitToBranchTakesTheBranchSnapshot) ASSERT_OK(branch_schema_manager.CreateTable(arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); - TableLoadOptions base_builder_load_options{table_path_}; + TableLoadContext base_builder_load_context{table_path_}; std::map base_builder_options; CommitContextBuilder base_builder("commit_user_1"); base_builder_options[Options::FILE_SYSTEM] = "local"; base_builder_options[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr base_commit, - FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + FileStoreCommit::Create(base_builder_load_context, std::move(base_context), base_builder_options)); ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(1), /*commit_identifier=*/1)); SnapshotManager branch_snapshot_manager(file_system_, table_path_, "dev"); @@ -1163,13 +1163,13 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWinsOverTheOneUnderTheTablePath ASSERT_EQ(catalog->CommitCalls().size(), 1u); ASSERT_EQ(catalog->CommitCalls().front().snapshot.SchemaId(), 7); - TableLoadOptions path_builder_load_options{table_path_}; + TableLoadContext path_builder_load_context{table_path_}; std::map path_builder_options; CommitContextBuilder path_builder("commit_user_2"); path_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, path_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto path_commit, - FileStoreCommit::Create(path_builder_load_options, std::move(path_context), + FileStoreCommit::Create(path_builder_load_context, std::move(path_context), path_builder_options)); std::vector> other_msgs = CommitMessagesOfRound(2); ASSERT_OK(path_commit->Commit(other_msgs, 2)); @@ -1237,13 +1237,13 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSchemaWhichIsNotADataTableSchemaIsRef } TEST_F(FileStoreCommitImplTest, TestTableIdWithNothingToSendItInIsRefused) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; - context_builder_load_options.table_id = "table-uuid"; + context_builder_load_context.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(context_builder_load_options, + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options), "a table id requires a catalog or an external REST commit request"); @@ -1416,16 +1416,16 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithoutAUsableLockIsNotCreated) { ASSERT_NOK_WITH_MSG(CreateCatalogCommit(catalog), "no commit lease for this table"); ASSERT_TRUE(catalog->CommitCalls().empty()); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder("commit_user_1"); builder_options[Options::FILE_SYSTEM] = "local"; builder_options[Options::REST_COMMIT_ACQUIRE_LOCK] = "true"; - builder_load_options.table_id = "table-uuid"; + builder_load_context.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.UseRESTCatalogCommit(true).Finish()); ASSERT_NOK_WITH_MSG( - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options), + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options), "cannot hold a commit lease"); } @@ -1722,14 +1722,14 @@ TEST_F(FileStoreCommitImplTest, TestCommitReadsThroughTheCatalogsFileSystem) { auto catalog = CreateCatalog(); catalog->SetFileSystem(fs); - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); - context_builder_load_options.catalog = catalog; - context_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + context_builder_load_context.catalog = catalog; + context_builder_load_context.identifier.emplace(Identifier("db", "tbl")); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); @@ -1763,15 +1763,15 @@ TEST_F(FileStoreCommitImplTest, TestAGivenFileSystemWinsOverTheCatalogs) { auto catalog = CreateCatalog(); catalog->SetFileSystem(catalog_fs); - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); - context_builder_load_options.catalog = catalog; - context_builder_load_options.identifier.emplace(Identifier("db", "tbl")); - context_builder_load_options.file_system = given_fs; + context_builder_load_context.catalog = catalog; + context_builder_load_context.identifier.emplace(Identifier("db", "tbl")); + context_builder_load_context.file_system = given_fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); ASSERT_OK(commit->Commit(CommitMessagesOfRound(1))); @@ -2067,7 +2067,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogSnapshotLoaderFailurePropagates) { } TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -2078,7 +2078,7 @@ TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -2119,20 +2119,20 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::COMMIT_MAX_RETRIES] = "10"; context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; - context_builder_load_options.file_system = fs; + context_builder_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); @@ -2165,19 +2165,19 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; context_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; - context_builder_load_options.file_system = fs; + context_builder_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); @@ -2243,7 +2243,7 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); - TableLoadOptions realtime_builder_load_options{table_path_}; + TableLoadContext realtime_builder_load_context{table_path_}; std::map realtime_builder_options; CommitContextBuilder realtime_builder("realtime_commit_user"); realtime_builder_options[Options::REALTIME_ENABLED] = "true"; @@ -2251,13 +2251,13 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) realtime_builder_options[Options::COMMIT_MAX_RETRIES] = "1"; realtime_builder_options[Options::COMMIT_MIN_RETRY_WAIT] = "1ms"; realtime_builder_options[Options::COMMIT_MAX_RETRY_WAIT] = "1ms"; - realtime_builder_load_options.file_system = fs; + realtime_builder_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_context, realtime_builder .Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr realtime_commit, - FileStoreCommit::Create(realtime_builder_load_options, std::move(realtime_context), + FileStoreCommit::Create(realtime_builder_load_context, std::move(realtime_context), realtime_builder_options)); auto realtime_commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(realtime_commit))); @@ -2275,15 +2275,15 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict) OffsetRange(/*begin=*/0, /*end=*/1)}); } - TableLoadOptions concurrent_builder_load_options{table_path_}; + TableLoadContext concurrent_builder_load_context{table_path_}; std::map concurrent_builder_options; CommitContextBuilder concurrent_builder("concurrent_commit_user"); - concurrent_builder_load_options.file_system = file_system_; + concurrent_builder_load_context.file_system = file_system_; ASSERT_OK_AND_ASSIGN(std::unique_ptr concurrent_context, concurrent_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr concurrent_commit, - FileStoreCommit::Create(concurrent_builder_load_options, std::move(concurrent_context), + FileStoreCommit::Create(concurrent_builder_load_context, std::move(concurrent_context), concurrent_builder_options)); auto* mock_fs = dynamic_cast(fs.get()); @@ -2328,15 +2328,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path, {})); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; - context_builder_load_options.file_system = fs; + context_builder_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::string new_snapshot_6 = PathUtil::JoinPath(table_path, "snapshot/snapshot-6"); auto* mock_fs = dynamic_cast(fs.get()); @@ -2358,15 +2358,15 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua bool exist, file_system_->Exists(PathUtil::JoinPath(table_path, "snapshot/snapshot-6"))); ASSERT_TRUE(exist); - TableLoadOptions context_builder_2_load_options{table_path}; + TableLoadContext context_builder_2_load_context{table_path}; std::map context_builder_2_options; CommitContextBuilder context_builder_2("commit_user_1"); context_builder_2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; - context_builder_2_load_options.file_system = fs; + context_builder_2_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context_2, context_builder_2.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(context_builder_2_load_options, + ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(context_builder_2_load_context, std::move(commit_context_2), context_builder_2_options)); ASSERT_OK_AND_ASSIGN(int32_t num_committed, commit_2->FilterAndCommit({{1, msgs}})); @@ -2386,7 +2386,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua } TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "5kb"; @@ -2397,7 +2397,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); { @@ -2454,7 +2454,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { } TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -2462,7 +2462,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); { @@ -2528,14 +2528,14 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2588,13 +2588,13 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { - TableLoadOptions base_builder_load_options{table_path_}; + TableLoadContext base_builder_load_context{table_path_}; std::map base_builder_options; CommitContextBuilder base_builder("commit_user_1"); base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto base_commit, - FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + FileStoreCommit::Create(base_builder_load_context, std::move(base_context), base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); @@ -2619,13 +2619,13 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestThroughCatalog) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { - TableLoadOptions base_builder_load_options{table_path_}; + TableLoadContext base_builder_load_context{table_path_}; std::map base_builder_options; CommitContextBuilder base_builder("commit_user_1"); base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto base_commit, - FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + FileStoreCommit::Create(base_builder_load_context, std::move(base_context), base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); @@ -2655,13 +2655,13 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestHoldsALockScope) { } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTablePath) { - TableLoadOptions base_builder_load_options{table_path_}; + TableLoadContext base_builder_load_context{table_path_}; std::map base_builder_options; CommitContextBuilder base_builder("commit_user_1"); base_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr base_context, base_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto base_commit, - FileStoreCommit::Create(base_builder_load_options, std::move(base_context), + FileStoreCommit::Create(base_builder_load_context, std::move(base_context), base_builder_options)); for (int32_t round = 1; round <= 2; ++round) { ASSERT_OK(base_commit->Commit(CommitMessagesOfRound(round), round)); @@ -2676,28 +2676,28 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNeedsTheTargetUnderTheTabl } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->RollbackToAsLatest(/*target_snapshot_id=*/1), "Latest snapshot is null, can not roll back."); } TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -2727,7 +2727,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) /*ignore_if_exists=*/false)); std::string dv_table_path = PathUtil::JoinPath(test_root_, "foo.db/dv_bar"); - TableLoadOptions context_builder_load_options{dv_table_path}; + TableLoadContext context_builder_load_context{dv_table_path}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -2738,7 +2738,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2809,14 +2809,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF // before writing. A stateful Exists() mock reproduces exactly that ordering. ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, FileSystemFactory::Get("gmock_fs", table_path_, {})); - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; - context_builder_load_options.file_system = fs; + context_builder_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -2854,14 +2854,14 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF } TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); { std::vector> msgs = @@ -2902,14 +2902,14 @@ TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { } TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = @@ -2962,7 +2962,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { } TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -2970,7 +2970,7 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); { @@ -3083,7 +3083,7 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3092,7 +3092,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs; ASSERT_OK(commit->Commit(msgs)); @@ -3104,7 +3104,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { } TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3113,7 +3113,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -3137,7 +3137,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3146,7 +3146,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3174,7 +3174,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3183,7 +3183,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3204,7 +3204,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { } TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3213,7 +3213,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3539,7 +3539,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogExpireWithoutPublishedSnapshots) { } TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3554,7 +3554,7 @@ TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) { .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -3616,7 +3616,7 @@ TEST_F(FileStoreCommitImplTest, TestExpireOnMainOfTableWithBranchesIsNotSupporte } TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3631,7 +3631,7 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -3668,7 +3668,7 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { } TEST_F(FileStoreCommitImplTest, TestTruncateTable) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3677,7 +3677,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateTable) { context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -3704,13 +3704,13 @@ TEST_F(FileStoreCommitImplTest, TestTruncateTable) { } TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3760,13 +3760,13 @@ TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { } TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3788,13 +3788,13 @@ TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) { // A delete that fails with an IO error (e.g. permission denied) must be swallowed by // best-effort cleanup, mirroring Java deleteQuietly: Abort still returns OK and does not // propagate the failure. - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -3831,7 +3831,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { // Truncating a table that has never been committed succeeds and produces an OVERWRITE snapshot // with no added and no deleted files: there is nothing to overwrite, but the overwrite is still // materialized. - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3839,7 +3839,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); ASSERT_OK(commit->TruncateTable(/*commit_identifier=*/1)); @@ -3859,7 +3859,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { } TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3868,7 +3868,7 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3883,7 +3883,7 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { TEST_F(FileStoreCommitImplTest, TestCollectChanges) { ASSERT_OK(ConfigureTableOptions({{Options::BUCKET, "10"}})); - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3894,7 +3894,7 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + @@ -3931,7 +3931,7 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { } TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3939,7 +3939,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -3965,7 +3965,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3973,7 +3973,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4008,7 +4008,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { } TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4016,7 +4016,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4042,7 +4042,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) } TEST_F(FileStoreCommitImplTest, FilterAndCommit) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4050,7 +4050,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommit) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4092,7 +4092,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommit) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4100,7 +4100,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4117,7 +4117,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4125,7 +4125,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4154,7 +4154,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { } TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4162,7 +4162,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4190,7 +4190,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { } TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4198,7 +4198,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4232,7 +4232,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4240,7 +4240,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4268,7 +4268,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { ASSERT_OK(ConfigureTableOptions( {{Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::ROW_TRACKING_ENABLED, "true"}})); - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4280,7 +4280,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4305,7 +4305,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4314,7 +4314,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4341,7 +4341,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4349,7 +4349,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4382,7 +4382,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4391,7 +4391,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { context_builder.IgnoreEmptyCommit(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4425,7 +4425,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4433,7 +4433,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4462,7 +4462,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4470,7 +4470,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4520,7 +4520,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { } TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4528,7 +4528,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4562,7 +4562,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4570,7 +4570,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4599,7 +4599,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4607,7 +4607,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4632,7 +4632,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { } TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4643,7 +4643,7 @@ TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -4669,14 +4669,14 @@ TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { } TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = @@ -4704,7 +4704,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - TableLoadOptions context_builder2_load_options{tmp_table_path}; + TableLoadContext context_builder2_load_context{tmp_table_path}; std::map context_builder2_options; CommitContextBuilder context_builder2("commit_user_1"); context_builder2_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4712,7 +4712,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context2, context_builder2.Finish()); auto commit2 = FileStoreCommit::Create( - context_builder2_load_options, std::move(commit_context2), context_builder2_options); + context_builder2_load_context, std::move(commit_context2), context_builder2_options); CHECK_HOOK_STATUS(commit2.status(), i); CHECK_HOOK_STATUS(commit2.value()->Commit(msgs), i); commit_run_complete = true; @@ -4736,11 +4736,11 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_TRUE(is_oss); // REST commit with local path should work without the object store flag - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.UseRESTCatalogCommit(true).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(builder_load_options, std::move(ctx), + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(builder_load_context, std::move(ctx), builder_options)); std::vector> msgs = CommitMessagesOfRound(1); @@ -4749,14 +4749,14 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_FALSE(json.empty()); auto catalog = CreateCatalog(); - TableLoadOptions catalog_builder_load_options{table_path_}; + TableLoadContext catalog_builder_load_context{table_path_}; std::map catalog_builder_options; CommitContextBuilder catalog_builder("commit_user_1"); - catalog_builder_load_options.catalog = catalog; - catalog_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + catalog_builder_load_context.catalog = catalog; + catalog_builder_load_context.identifier.emplace(Identifier("db", "tbl")); ASSERT_OK_AND_ASSIGN(auto catalog_ctx, catalog_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto catalog_commit, - FileStoreCommit::Create(catalog_builder_load_options, + FileStoreCommit::Create(catalog_builder_load_context, std::move(catalog_ctx), catalog_builder_options)); ASSERT_OK(catalog_commit->Commit(msgs)); ASSERT_EQ(catalog->CommitCalls().size(), 1u); @@ -4766,21 +4766,21 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_TRUE(path_is_object_store); auto object_store_catalog = CreateCatalog(); object_store_catalog->SetFileSystem(file_system_); - TableLoadOptions object_store_builder_load_options{object_store_path}; + TableLoadContext object_store_builder_load_context{object_store_path}; std::map object_store_builder_options; CommitContextBuilder object_store_builder("commit_user_1"); - object_store_builder_load_options.catalog = object_store_catalog; - object_store_builder_load_options.identifier.emplace(Identifier("db", "tbl")); + object_store_builder_load_context.catalog = object_store_catalog; + object_store_builder_load_context.identifier.emplace(Identifier("db", "tbl")); ASSERT_OK_AND_ASSIGN(auto object_store_ctx, object_store_builder.Finish()); - ASSERT_OK(FileStoreCommit::Create(object_store_builder_load_options, + ASSERT_OK(FileStoreCommit::Create(object_store_builder_load_context, std::move(object_store_ctx), object_store_builder_options)); - TableLoadOptions plain_builder_load_options{object_store_path}; + TableLoadContext plain_builder_load_context{object_store_path}; std::map plain_builder_options; CommitContextBuilder plain_builder("commit_user_1"); - plain_builder_load_options.file_system = file_system_; + plain_builder_load_context.file_system = file_system_; ASSERT_OK_AND_ASSIGN(auto plain_ctx, plain_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(plain_builder_load_options, std::move(plain_ctx), + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(plain_builder_load_context, std::move(plain_ctx), plain_builder_options), "invalid scheme oss for local file system"); } @@ -4803,7 +4803,7 @@ TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) { std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - TableLoadOptions builder_load_options{pk_table_path}; + TableLoadContext builder_load_context{pk_table_path}; std::map builder_options; CommitContextBuilder builder("test_user"); builder_options[Options::FILE_SYSTEM] = "local"; @@ -4811,7 +4811,7 @@ TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) { ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( auto committer, - FileStoreCommit::Create(builder_load_options, std::move(commit_context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(commit_context), builder_options)); ASSERT_TRUE(committer != nullptr); } @@ -4841,7 +4841,7 @@ TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsAllowsManifestDeleteFileDro } TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4852,7 +4852,7 @@ TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { .Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4875,7 +4875,7 @@ TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) { } TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -4886,7 +4886,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { .Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -4955,27 +4955,27 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) { } TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); ASSERT_NOK_WITH_MSG(commit->DropPartition({}, /*commit_identifier=*/1), "partitions list cannot be empty"); } TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5010,13 +5010,13 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5028,13 +5028,13 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) } TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5059,7 +5059,7 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { } TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -5069,7 +5069,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5092,14 +5092,14 @@ TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) } TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5140,7 +5140,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - TableLoadOptions builder_load_options{pk_table_path}; + TableLoadContext builder_load_context{pk_table_path}; std::map builder_options; CommitContextBuilder builder("test_user"); builder_options[Options::FILE_SYSTEM] = "local"; @@ -5148,7 +5148,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( auto commit, - FileStoreCommit::Create(builder_load_options, std::move(commit_context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5193,7 +5193,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) } TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -5201,7 +5201,7 @@ TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.AppendCommitCheckConflict(true).Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs = @@ -5219,14 +5219,14 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // First snapshot is committed in the default (scan) mode, so it does NOT carry the // max-sequence-number property. { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); std::vector> msgs1 = @@ -5238,7 +5238,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { // Second commit uses snapshot-init mode; since the previous snapshot lacks the property, // the max sequence number is recomputed by scanning the base manifests. - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -5248,7 +5248,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { .Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5272,14 +5272,14 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { } TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5301,14 +5301,14 @@ TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { } TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; context_builder_options[Options::OVERWRITE_UPGRADE] = "false"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5337,7 +5337,7 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - TableLoadOptions builder_load_options{pk_table_path}; + TableLoadContext builder_load_context{pk_table_path}; std::map builder_options; CommitContextBuilder builder("test_user"); builder_options[Options::FILE_SYSTEM] = "local"; @@ -5345,7 +5345,7 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( auto commit, - FileStoreCommit::Create(builder_load_options, std::move(commit_context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(commit_context), builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5360,14 +5360,14 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { } TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5386,13 +5386,13 @@ TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelt } TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); @@ -5403,13 +5403,13 @@ TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests } TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) { - TableLoadOptions context_builder_load_options{table_path_}; + TableLoadContext context_builder_load_context{table_path_}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user_1"); context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index e034167f7..ace1a6b08 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -29,7 +29,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/utils/linked_hash_map.h" #include "paimon/common/utils/path_util.h" @@ -81,14 +81,14 @@ TEST(FileStoreCommitTest, TestCreate) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); auto commit_impl = dynamic_cast(commit.get()); ASSERT_TRUE(commit_impl); @@ -99,11 +99,11 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { ASSERT_TRUE(dir); std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir->Str(), options)); - TableLoadOptions load_options{dir->Str()}; - load_options.catalog = catalog; + TableLoadContext load_context{dir->Str()}; + load_context.catalog = catalog; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, CommitContextBuilder("commit_user").Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(load_options, std::move(context), options), + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(load_context, std::move(context), options), "a catalog load requires a table identifier"); } @@ -125,13 +125,13 @@ TEST(FileStoreCommitTest, TestCatalogCommitUsesPerTableFileSystem) { catalog->SetFileSystem(std::make_shared()); catalog->SetTableFileSystem(dir->GetFileSystem()); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; CommitContextBuilder builder("commit_user"); - builder_load_options.catalog = catalog; - builder_load_options.identifier.emplace(identifier); + builder_load_context.catalog = catalog; + builder_load_context.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(builder_load_options, std::move(ctx), + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(builder_load_context, std::move(ctx), builder_options)); // The file system was resolved per table, not from the catalog-wide one. @@ -167,14 +167,14 @@ TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; CommitContextBuilder context_builder("commit_user"); context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; context_builder_options[Options::FILE_SYSTEM] = "local"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(context_builder_load_options, + auto commit, FileStoreCommit::Create(context_builder_load_context, std::move(commit_context), context_builder_options)); LinkedHashMap dv_ranges; diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 4ce2e7341..2a2419b74 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -32,7 +32,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" @@ -89,16 +89,16 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { }; auto create_writer = [&](const std::shared_ptr& realtime_context) -> Result> { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder("writer"); - builder_load_options.catalog = catalog; - builder_load_options.identifier.emplace(identifier); + builder_load_context.catalog = catalog; + builder_load_context.identifier.emplace(identifier); builder_options[Options::FILE_SYSTEM] = "unregistered-file-system"; PAIMON_ASSIGN_OR_RAISE( std::unique_ptr ctx, builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).Finish()); - return FileStoreWrite::Create(builder_load_options, std::move(ctx), builder_options); + return FileStoreWrite::Create(builder_load_context, std::move(ctx), builder_options); }; ASSERT_OK_AND_ASSIGN(auto realtime_context, RealtimeContext::Create()); ASSERT_OK_AND_ASSIGN(auto writer, create_writer(realtime_context)); @@ -109,14 +109,14 @@ TEST(FileStoreWriteTest, TestRealtimeCatalogCommitRefreshAndRecovery) { ASSERT_EQ(base_snapshot_id, std::nullopt); ASSERT_OK_AND_ASSIGN(auto progress, writer->PrepareCommitWithProgress(1)); ASSERT_EQ(progress.size(), 1u); - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("writer"); - commit_builder_load_options.catalog = catalog; - commit_builder_load_options.identifier.emplace(identifier); + commit_builder_load_context.catalog = catalog; + commit_builder_load_context.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(auto commit_ctx, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto commit, FileStoreCommit::Create(commit_builder_load_options, std::move(commit_ctx), + auto commit, FileStoreCommit::Create(commit_builder_load_context, std::move(commit_ctx), commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, commit->CommitWithProgress(progress, 1, std::nullopt)); @@ -198,15 +198,15 @@ TEST(FileStoreWriteTest, TestBucketBaseSnapshotTracksActualLazyRestore) { catalog->SetFileSystem(dir->GetFileSystem()); catalog->CheckBaseSnapshotUuid(); auto create_writer = [&](bool ignore_previous) -> Result> { - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; WriteContextBuilder builder("writer"); - builder_load_options.catalog = catalog; - builder_load_options.identifier.emplace(identifier); + builder_load_context.catalog = catalog; + builder_load_context.identifier.emplace(identifier); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr context, builder.WithStreamingMode(true).WithIgnorePreviousFiles(ignore_previous).Finish()); - return FileStoreWrite::Create(builder_load_options, std::move(context), + return FileStoreWrite::Create(builder_load_context, std::move(context), builder_options); }; auto make_batch = [&](const std::string& dt, int32_t bucket, @@ -221,13 +221,13 @@ TEST(FileStoreWriteTest, TestBucketBaseSnapshotTracksActualLazyRestore) { .SetBucket(bucket) .Finish(); }; - TableLoadOptions commit_builder_load_options{dir->Str()}; + TableLoadContext commit_builder_load_context{dir->Str()}; std::map commit_builder_options; CommitContextBuilder commit_builder("writer"); - commit_builder_load_options.catalog = catalog; - commit_builder_load_options.identifier.emplace(identifier); + commit_builder_load_context.catalog = catalog; + commit_builder_load_context.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); auto seed_snapshot = [&](int32_t bucket, int64_t identifier) -> Status { @@ -291,15 +291,15 @@ TEST(FileStoreWriteTest, TestBucketBaseSnapshotTracksActualLazyRestore) { TEST(FileStoreWriteTest, TestCreateWithInvalidInput) { auto dir = UniqueTestDirectory::Create(); - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; WriteContextBuilder builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx1, builder.WithMemoryPool(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(builder_load_options, std::move(ctx1), builder_options)); + ASSERT_NOK(FileStoreWrite::Create(builder_load_context, std::move(ctx1), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx2, builder.WithExecutor(nullptr).Finish()); - ASSERT_NOK(FileStoreWrite::Create(builder_load_options, std::move(ctx2), builder_options)); + ASSERT_NOK(FileStoreWrite::Create(builder_load_context, std::move(ctx2), builder_options)); ASSERT_NOK(FileStoreWrite::Create({}, /*context=*/nullptr)); } @@ -321,17 +321,17 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { ASSERT_FALSE(shared_catalog->SupportsVersionManagement()); for (bool use_catalog : {false, true}) { SCOPED_TRACE(use_catalog); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); if (use_catalog) { - context_builder_load_options.catalog = shared_catalog; - context_builder_load_options.identifier.emplace(Identifier("foo", "bar")); + context_builder_load_context.catalog = shared_catalog; + context_builder_load_context.identifier.emplace(Identifier("foo", "bar")); } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); ASSERT_OK(file_store_write->Close()); } @@ -349,14 +349,14 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem catalog->SetTableSchema(schema); catalog->SetFileSystem(dir->GetFileSystem()); - TableLoadOptions main_builder_load_options{dir->Str()}; + TableLoadContext main_builder_load_context{dir->Str()}; std::map main_builder_options; WriteContextBuilder main_builder("commit_user_1"); - main_builder_load_options.catalog = catalog; - main_builder_load_options.identifier.emplace(Identifier("db", "table$branch_main")); + main_builder_load_context.catalog = catalog; + main_builder_load_context.identifier.emplace(Identifier("db", "table$branch_main")); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_write, - FileStoreWrite::Create(main_builder_load_options, std::move(main_context), + FileStoreWrite::Create(main_builder_load_context, std::move(main_context), main_builder_options)); ASSERT_OK(main_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); @@ -366,14 +366,14 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem ASSERT_OK(branch_schema_manager.CreateTable(logical_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); catalog->SetTableSchema(nullptr); - TableLoadOptions branch_builder_load_options{dir->Str()}; + TableLoadContext branch_builder_load_context{dir->Str()}; std::map branch_builder_options; WriteContextBuilder branch_builder("commit_user_1"); - branch_builder_load_options.catalog = catalog; - branch_builder_load_options.identifier.emplace(Identifier("db", "table$branch_dev")); + branch_builder_load_context.catalog = catalog; + branch_builder_load_context.identifier.emplace(Identifier("db", "table$branch_dev")); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_write, - FileStoreWrite::Create(branch_builder_load_options, + FileStoreWrite::Create(branch_builder_load_context, std::move(branch_context), branch_builder_options)); ASSERT_OK(branch_write->Close()); ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); @@ -397,15 +397,15 @@ TEST(FileStoreWriteTest, TestCatalogWriteUsesPerTableFileSystem) { catalog->SetFileSystem(std::make_shared()); catalog->SetTableFileSystem(dir->GetFileSystem()); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder("writer"); - builder_load_options.catalog = catalog; - builder_load_options.identifier.emplace(identifier); + builder_load_context.catalog = catalog; + builder_load_context.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr writer, - FileStoreWrite::Create(builder_load_options, std::move(ctx), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(ctx), builder_options)); ASSERT_OK(writer->Close()); // The file system was resolved per table, not from the catalog-wide one. @@ -429,12 +429,12 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidBucket) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options[Options::BUCKET] = "-2"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options), "Cannot change bucket from or to -1 dynamically"); } @@ -457,12 +457,12 @@ TEST(FileStoreWriteTest, TestCreateAppendTableWithInvalidWriteType) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.WithWriteSchema({"field_non_exist"}).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options), "write field field_non_exist does not exist in table schema"); } @@ -483,13 +483,13 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0"}, /*primary_keys=*/{"f0", "f1", "f4"}, options, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options[Options::BUCKET] = "2"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); auto fs = std::make_shared(); @@ -519,11 +519,11 @@ TEST(FileStoreWriteTest, TestCreatePrimaryKeyTableWithInvalidBucket) { /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{"f1", "f4"}, options, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_options, + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options), "not support bucket -1 in key value table"); } diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index b393ae69e..9d1d84826 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -29,7 +29,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/catalog_options.h" #include "paimon/common/catalog/catalog_context.h" #include "paimon/common/table/special_fields.h" @@ -53,11 +53,11 @@ namespace paimon::test { TEST(InternalReadContext, TestReadWithUnspecifiedSchema) { // no read schema is specified, read all fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -77,7 +77,7 @@ TEST(InternalReadContext, TestPreciseScalarEvaluation) { arrow::field("f1", arrow::utf8())}), {}, {}, {})); for (bool enabled : {false, true}) { - TableLoadOptions builder_load_options{"unused-table-path"}; + TableLoadContext builder_load_context{"unused-table-path"}; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(enabled).SetPredicate( @@ -86,9 +86,9 @@ TEST(InternalReadContext, TestPreciseScalarEvaluation) { ASSERT_OK_AND_ASSIGN( std::shared_ptr internal_context, InternalReadContext::Create( - context, table_schema, table_schema->Options(), builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + context, table_schema, table_schema->Options(), builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); ASSERT_EQ(internal_context->EnablePreciseFileIndexEvaluate(), enabled); ASSERT_FALSE(internal_context->EnablePredicateFilter()); ASSERT_TRUE(internal_context->GetCoreOptions().FileIndexReadEnabled()); @@ -100,7 +100,7 @@ TEST(InternalReadContext, TestPreciseScalarEvaluation) { } auto match = std::make_shared("f1", R"({"match":{"query":"apple"}})"); ASSERT_OK_AND_ASSIGN(auto full_text, PredicateBuilder::FullTextMatch(1, match)); - TableLoadOptions builder_load_options{"unused-table-path"}; + TableLoadContext builder_load_context{"unused-table-path"}; std::map builder_options; ReadContextBuilder builder; builder.SetPredicate(full_text); @@ -109,16 +109,16 @@ TEST(InternalReadContext, TestPreciseScalarEvaluation) { ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_TRUE(context->EnablePreciseFileIndexEvaluate()); ASSERT_OK(InternalReadContext::Create( - context, table_schema, table_schema->Options(), builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + context, table_schema, table_schema->Options(), builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); } TEST(InternalReadContext, TestPreciseEvaluationRequiresFileIndexRead) { ASSERT_OK_AND_ASSIGN( std::shared_ptr table_schema, TableSchema::Create(0, arrow::schema({arrow::field("f0", arrow::int32())}), {}, {}, {})); - TableLoadOptions builder_load_options{"unused-table-path"}; + TableLoadContext builder_load_context{"unused-table-path"}; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); @@ -126,20 +126,20 @@ TEST(InternalReadContext, TestPreciseEvaluationRequiresFileIndexRead) { ASSERT_NOK_WITH_MSG( InternalReadContext::Create( context, table_schema, {{Options::FILE_INDEX_READ_ENABLED, "false"}}, - builder_load_options.file_system, builder_load_options.path, - builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, - builder_load_options.identifier, builder_options), + builder_load_context.file_system, builder_load_context.path, + builder_load_context.fs_scheme_to_identifier_map, builder_load_context.catalog, + builder_load_context.identifier, builder_options), "requires file-index.read.enabled=true"); } TEST(InternalReadContext, TestReadWithSpecifiedSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -167,11 +167,11 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { {CatalogOptions::URI, "http://read"}, {CatalogOptions::TOKEN, "read-token"}, {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}}; - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; - builder_load_options.catalog = catalog; - builder_load_options.identifier.emplace(Identifier("db1", "t1")); + builder_load_context.catalog = catalog; + builder_load_context.identifier.emplace(Identifier("db1", "t1")); builder_options = read_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); std::map options = table_schema->Options(); @@ -184,9 +184,9 @@ TEST(InternalReadContext, TestDependencyReadMergesTableCatalogAndReadOptions) { ASSERT_OK_AND_ASSIGN( auto internal_context, InternalReadContext::Create( - read_context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); auto expected_options = options; @@ -223,11 +223,11 @@ TEST(InternalReadContext, TestDependencyReadKeepsTheOutermostView) { if (test_case.read_header) { read_options["header.X-Paimon-Read-Via"] = test_case.read_header; } - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; - builder_load_options.catalog = catalog; - builder_load_options.identifier.emplace(Identifier("db1", "nested-view")); + builder_load_context.catalog = catalog; + builder_load_context.identifier.emplace(Identifier("db1", "nested-view")); builder_options = read_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); auto options = table_schema->Options(); @@ -236,9 +236,9 @@ TEST(InternalReadContext, TestDependencyReadKeepsTheOutermostView) { ASSERT_OK_AND_ASSIGN( auto internal_context, InternalReadContext::Create( - read_context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); ASSERT_EQ(dependency_context->options.at("header.X-Paimon-Read-Via"), test_case.expected_header); @@ -286,19 +286,19 @@ TEST(InternalReadContext, TestDependencyReadHeaderUsesFinalMetastore) { read_options[CatalogOptions::METASTORE] = test_case.read_metastore; options[CatalogOptions::METASTORE] = test_case.read_metastore; } - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; - builder_load_options.catalog = catalog; - builder_load_options.identifier.emplace(Identifier("db1", "t1")); + builder_load_context.catalog = catalog; + builder_load_context.identifier.emplace(Identifier("db1", "t1")); builder_options = read_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( auto internal_context, InternalReadContext::Create( - read_context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + read_context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); auto dependency_context = internal_context->CreateDependencyCatalogContext(); if (test_case.expected_metastore) { ASSERT_EQ(dependency_context->options.at(CatalogOptions::METASTORE), @@ -321,10 +321,10 @@ TEST(InternalReadContext, TestDependencyReadHeaderNeedsAnIdentifier) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; ReadContextBuilder builder; ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); - ASSERT_FALSE(builder_load_options.identifier); + ASSERT_FALSE(builder_load_context.identifier); auto options = table_schema->Options(); options[CatalogOptions::METASTORE] = "REST"; ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), @@ -359,13 +359,13 @@ TEST(InternalReadContext, TestFileSystemSchemeMapReachesTheDependencyRead) { // surfaces as a `dfs://` dependency table that cannot be opened, far away from the cause. std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; const std::map scheme_map = {{"file", "local"}}; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; - context_builder_load_options.fs_scheme_to_identifier_map = scheme_map; + context_builder_load_context.fs_scheme_to_identifier_map = scheme_map; ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, @@ -378,12 +378,12 @@ TEST(InternalReadContext, TestFileSystemSchemeMapReachesTheDependencyRead) { TEST(InternalReadContext, TestReadWithSpecifiedFieldId) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; context_builder.SetReadFieldIds({3, 0}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -396,7 +396,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldId) { TEST(InternalReadContext, TestReadWithSpecifiedFieldIdAndSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; // read schema is specified, read fields in schema // will use field ids instead of field names. @@ -404,7 +404,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldIdAndSchema) { context_builder.SetReadFieldIds({3, 0}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -419,13 +419,13 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { { // test simple std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; context_builder.SetReadFieldNames( {"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER", "_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); auto new_options = table_schema->Options(); new_options[Options::ROW_TRACKING_ENABLED] = "true"; @@ -443,12 +443,12 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { { // test invalid case: disable row tracking while read row tracking fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options()), @@ -457,12 +457,12 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { { // test invalid case: disable data evolution while read score fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_NOK_WITH_MSG(InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options()), @@ -477,7 +477,7 @@ TEST(InternalReadContext, TestPreciseEvaluationRejectsUnnamedScoreProjection) { for (const auto& search_type : std::vector{"none", "scalar", "vector", "full-text", "full-text-match"}) { for (const auto& projection : std::vector{"names", "ids", "schema"}) { - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f0"}); @@ -515,9 +515,9 @@ TEST(InternalReadContext, TestPreciseEvaluationRejectsUnnamedScoreProjection) { ASSERT_NOK_WITH_MSG( InternalReadContext::Create( context, table_schema, table_schema->Options(), - builder_load_options.file_system, builder_load_options.path, - builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, - builder_load_options.identifier, builder_options), + builder_load_context.file_system, builder_load_context.path, + builder_load_context.fs_scheme_to_identifier_map, builder_load_context.catalog, + builder_load_context.identifier, builder_options), "not exist in table schema"); } } @@ -531,7 +531,7 @@ TEST(InternalReadContext, TestPrimaryKeyVectorSearchRequiresDeletionVectors) { std::shared_ptr search = std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder.SetReadFieldNames({"f0", "_f3_INDEX_SCORE"}) @@ -541,9 +541,9 @@ TEST(InternalReadContext, TestPrimaryKeyVectorSearchRequiresDeletionVectors) { ASSERT_OK_AND_ASSIGN( std::unique_ptr internal, InternalReadContext::Create( - context, table_schema, table_schema->Options(), builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options)); + context, table_schema, table_schema->Options(), builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options)); EXPECT_TRUE(internal->HasVectorOrFullTextSearch()); EXPECT_TRUE(internal->EnablePreciseFileIndexEvaluate()); @@ -551,9 +551,9 @@ TEST(InternalReadContext, TestPrimaryKeyVectorSearchRequiresDeletionVectors) { options[Options::DELETION_VECTORS_ENABLED] = "false"; ASSERT_NOK_WITH_MSG( InternalReadContext::Create( - context, table_schema, options, builder_load_options.file_system, - builder_load_options.path, builder_load_options.fs_scheme_to_identifier_map, - builder_load_options.catalog, builder_load_options.identifier, builder_options), + context, table_schema, options, builder_load_context.file_system, + builder_load_context.path, builder_load_context.fs_scheme_to_identifier_map, + builder_load_context.catalog, builder_load_context.identifier, builder_options), "primary-key tables requires deletion vectors"); } @@ -580,7 +580,7 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { std::make_shared(test_case.field_name, R"({"match":{"query":"apple"}})"); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch(test_case.field_index, match)); - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) @@ -592,9 +592,9 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { std::shared_ptr internal, InternalReadContext::Create( context, table_schema, table_schema->Options(), - builder_load_options.file_system, builder_load_options.path, - builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, - builder_load_options.identifier, builder_options)); + builder_load_context.file_system, builder_load_context.path, + builder_load_context.fs_scheme_to_identifier_map, builder_load_context.catalog, + builder_load_context.identifier, builder_options)); ASSERT_TRUE(internal->GetIndexScoreBindings().empty()); ASSERT_FALSE(internal->NeedsScores( {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH, match->field_name})); @@ -605,9 +605,9 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { ASSERT_NOK_WITH_MSG( InternalReadContext::Create( context, table_schema, table_schema->Options(), - builder_load_options.file_system, builder_load_options.path, - builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, - builder_load_options.identifier, builder_options), + builder_load_context.file_system, builder_load_context.path, + builder_load_context.fs_scheme_to_identifier_map, builder_load_context.catalog, + builder_load_context.identifier, builder_options), test_case.expected_error); } } @@ -621,7 +621,7 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { for (bool vector_search : {false, true}) { for (const auto& projection_type : std::vector{"none", "names", "schema"}) { const bool project_score = projection_type != "none"; - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; std::vector projection = {"f1"}; @@ -656,9 +656,9 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { std::shared_ptr internal, InternalReadContext::Create( context, table_schema, table_schema->Options(), - builder_load_options.file_system, builder_load_options.path, - builder_load_options.fs_scheme_to_identifier_map, builder_load_options.catalog, - builder_load_options.identifier, builder_options)); + builder_load_context.file_system, builder_load_context.path, + builder_load_context.fs_scheme_to_identifier_map, builder_load_context.catalog, + builder_load_context.identifier, builder_options)); ASSERT_EQ(internal->GetReadSchema()->field_names(), projection); ASSERT_TRUE(internal->NeedsScores(source)); ASSERT_FALSE(internal->NeedsScores( @@ -969,12 +969,12 @@ TEST_F(InternalReadContextScoreTest, SchemaReplacementUsesProvidedSchema) { TEST(InternalReadContext, TestReadWithValueKindField) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "_VALUE_KIND", "f0"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create(std::move(read_context), table_schema, @@ -990,7 +990,7 @@ TEST(InternalReadContext, TestReadWithFieldIdsAndSpecialFields) { { // test simple std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; ReadContextBuilder context_builder; // here we use field ids instead of field names, and specify special ids for row id, // sequence number and index score. @@ -999,7 +999,7 @@ TEST(InternalReadContext, TestReadWithFieldIdsAndSpecialFields) { SpecialFieldIds::INDEX_SCORE}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); auto new_options = table_schema->Options(); new_options[Options::ROW_TRACKING_ENABLED] = "true"; @@ -1030,7 +1030,7 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); @@ -1039,9 +1039,9 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { ASSERT_NOK_WITH_MSG( InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - context_builder_load_options.file_system, context_builder_load_options.path, - context_builder_load_options.fs_scheme_to_identifier_map, - context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_load_context.file_system, context_builder_load_context.path, + context_builder_load_context.fs_scheme_to_identifier_map, + context_builder_load_context.catalog, context_builder_load_context.identifier, context_builder_options), "not exist in table schema"); } @@ -1055,7 +1055,7 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); @@ -1065,9 +1065,9 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { auto internal_context, InternalReadContext::Create( read_context, table_schema, enabled_options, - context_builder_load_options.file_system, context_builder_load_options.path, - context_builder_load_options.fs_scheme_to_identifier_map, - context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_load_context.file_system, context_builder_load_context.path, + context_builder_load_context.fs_scheme_to_identifier_map, + context_builder_load_context.catalog, context_builder_load_context.identifier, context_builder_options)); auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); ASSERT_TRUE(internal_context->GetReadSchema()->Equals(expected_schema)); @@ -1082,7 +1082,7 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaWithoutFieldIds) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); @@ -1090,16 +1090,16 @@ TEST(InternalReadContext, TestReadWithProjectedSchemaWithoutFieldIds) { std::shared_ptr read_context = std::move(unique_read_context); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - context_builder_load_options.file_system, context_builder_load_options.path, - context_builder_load_options.fs_scheme_to_identifier_map, - context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_load_context.file_system, context_builder_load_context.path, + context_builder_load_context.fs_scheme_to_identifier_map, + context_builder_load_context.catalog, context_builder_load_context.identifier, context_builder_options)); std::vector expected_fields = { @@ -1122,7 +1122,7 @@ TEST(InternalReadContext, TestProjectedSchemaMetadataWhitelist) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_schema)); @@ -1130,16 +1130,16 @@ TEST(InternalReadContext, TestProjectedSchemaMetadataWhitelist) { std::shared_ptr read_context = std::move(unique_read_context); SchemaManager schema_manager(std::make_shared(), - context_builder_load_options.path); + context_builder_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - context_builder_load_options.file_system, context_builder_load_options.path, - context_builder_load_options.fs_scheme_to_identifier_map, - context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_load_context.file_system, context_builder_load_context.path, + context_builder_load_context.fs_scheme_to_identifier_map, + context_builder_load_context.catalog, context_builder_load_context.identifier, context_builder_options)); auto aligned_field = internal_context->GetReadSchema()->GetFieldByName("f0"); @@ -1175,7 +1175,7 @@ TEST(InternalReadContext, TestMapSharedShreddingAccessRequiresSharedShreddingLay auto c_read_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*arrow::schema({access_field}), c_read_schema.get()).ok()); - TableLoadOptions context_builder_load_options{"/tmp/unused-table-path"}; + TableLoadContext context_builder_load_context{"/tmp/unused-table-path"}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadSchema(std::move(c_read_schema)); @@ -1185,9 +1185,9 @@ TEST(InternalReadContext, TestMapSharedShreddingAccessRequiresSharedShreddingLay ASSERT_NOK_WITH_MSG( InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - context_builder_load_options.file_system, context_builder_load_options.path, - context_builder_load_options.fs_scheme_to_identifier_map, - context_builder_load_options.catalog, context_builder_load_options.identifier, + context_builder_load_context.file_system, context_builder_load_context.path, + context_builder_load_context.fs_scheme_to_identifier_map, + context_builder_load_context.catalog, context_builder_load_context.identifier, context_builder_options), "Selected-key MAP pushdown only supports top-level shared-shredding MAP field: tags"); } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index bb863ae12..bd1db4276 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -41,7 +41,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/data/shredding/map_shredding_defs.h" @@ -136,7 +136,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { /*primary_keys=*/{"f0"}, table_options, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("test"); if (with_temp_directory) { @@ -145,7 +145,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.Finish()); - return FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + return FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options); } @@ -200,14 +200,14 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { const std::string& table_path, const std::shared_ptr& schema, const std::map& options, const std::string& json, int64_t commit_identifier) const { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder("test"); builder_options = options; EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); EXPECT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(builder_load_options, std::move(write_context), + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( @@ -227,14 +227,14 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { void Commit(const std::string& table_path, const std::map& options, const std::vector>& commit_msgs) const { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; CommitContextBuilder builder("test"); builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_commit, - FileStoreCommit::Create(builder_load_options, + FileStoreCommit::Create(builder_load_context, std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); } @@ -352,12 +352,12 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{"f1"}, /*options=*/{{"bucket", "1"}}, /*ignore_if_exists=*/false)); - TableLoadOptions builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map builder_options; WriteContextBuilder builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(builder_load_options, std::move(write_context), + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); } @@ -373,13 +373,13 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{"f1"}, /*options=*/{{"bucket", "-2"}}, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); auto array = std::make_shared(); arrow::StringBuilder builder; @@ -408,13 +408,13 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { /*primary_keys=*/{"f1"}, /*options=*/{{"bucket", "2"}}, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); auto array = std::make_shared(); arrow::StringBuilder builder; @@ -466,7 +466,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder("test"); builder_options = options; @@ -476,7 +476,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr writer, - FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); std::unique_ptr batch = MakeBatch(realtime_schema, R"([ @@ -532,7 +532,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { SCOPED_TRACE(expected_error); ASSERT_OK_AND_ASSIGN(std::shared_ptr invalid_context, RealtimeContext::Create()); - TableLoadOptions invalid_builder_load_options{table_path}; + TableLoadContext invalid_builder_load_context{table_path}; std::map invalid_builder_options; WriteContextBuilder invalid_builder("invalid"); invalid_builder_options = options; @@ -543,7 +543,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) { invalid_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr invalid_writer, - FileStoreWrite::Create(invalid_builder_load_options, std::move(invalid_write_context), + FileStoreWrite::Create(invalid_builder_load_context, std::move(invalid_write_context), invalid_builder_options)); ASSERT_OK(invalid_writer->Write(MakeBatch(realtime_schema, R"([[30, 1, "existing"]])"))); ASSERT_OK_AND_ASSIGN(std::vector prepared, @@ -575,7 +575,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder("test"); builder_options = options; @@ -583,7 +583,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr writer, - FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); const int64_t allocations_before_write = pool->allocation_count; ASSERT_OK(writer->Write(MakeBatch(realtime_schema, R"([[0, 1, "one"]])"))); @@ -599,7 +599,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { std::shared_ptr rejecting_pool = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr rejecting_context, RealtimeContext::Create()); - TableLoadOptions rejecting_builder_load_options{table_path}; + TableLoadContext rejecting_builder_load_context{table_path}; std::map rejecting_builder_options; WriteContextBuilder rejecting_builder("rejecting"); rejecting_builder_options = options; @@ -610,7 +610,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) { rejecting_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr rejecting_writer, - FileStoreWrite::Create(rejecting_builder_load_options, std::move(rejecting_write_context), + FileStoreWrite::Create(rejecting_builder_load_context, std::move(rejecting_write_context), rejecting_builder_options)); ASSERT_OK(rejecting_writer->Write(MakeBatch(realtime_schema, "[]"))); const int64_t rejecting_allocations_before_write = rejecting_pool->allocation_count; @@ -641,7 +641,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_context, RealtimeContext::Create()); - TableLoadOptions initial_builder_load_options{table_path}; + TableLoadContext initial_builder_load_context{table_path}; std::map initial_builder_options; WriteContextBuilder initial_builder("initial"); initial_builder_options = options; @@ -650,7 +650,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { initial_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr initial_writer, - FileStoreWrite::Create(initial_builder_load_options, std::move(initial_write_context), + FileStoreWrite::Create(initial_builder_load_context, std::move(initial_write_context), initial_builder_options)); ASSERT_OK(initial_writer->Write(MakeBatch(realtime_schema, R"([[0, 0, "initial"]])"))); ASSERT_OK_AND_ASSIGN(std::vector initial_progress, @@ -663,7 +663,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { initial_message->GetNewFilesIncrement().NewFiles()[0]->AssignSequenceNumber(max - 2, max - 2); initial_progress[0].offset_range = OffsetRange(0, max - 1); - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("initial"); commit_builder_options = options; @@ -671,7 +671,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr committer, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, committer->CommitWithProgress(initial_progress, 0, std::nullopt)); @@ -679,7 +679,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder("boundary"); builder_options = options; @@ -687,7 +687,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) { ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr writer, - FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); ASSERT_OK( writer->Write(MakeBatch(realtime_schema, fmt::format(R"([[{}, 1, "legal"]])", max - 1)))); ASSERT_OK_AND_ASSIGN(auto store_rows, ReadRealtimePrimaryKeyStoreRows(realtime_context)); @@ -796,14 +796,14 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillSimple) { {Options::WRITE_BUFFER_SPILLABLE, "true"}}, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("test"); context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); auto key_value_file_store_write = dynamic_cast(file_store_write.get()); auto get_writer = [&](int32_t bucket) -> std::shared_ptr { @@ -859,7 +859,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { /*ignore_if_exists=*/false)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - TableLoadOptions first_context_builder_load_options{table_path}; + TableLoadContext first_context_builder_load_context{table_path}; std::map first_context_builder_options; WriteContextBuilder first_context_builder("first-writer"); first_context_builder_options = options; @@ -868,7 +868,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { first_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto first_write, - FileStoreWrite::Create(first_context_builder_load_options, std::move(first_context), + FileStoreWrite::Create(first_context_builder_load_context, std::move(first_context), first_context_builder_options)); ASSERT_OK(WriteSingleStringRow(first_write.get(), /*bucket=*/0, "alice")); ASSERT_OK_AND_ASSIGN(auto commit_messages, @@ -882,7 +882,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { SimpleStats::EmptyStats()); Commit(table_path, options, commit_messages); - TableLoadOptions restored_context_builder_load_options{table_path}; + TableLoadContext restored_context_builder_load_context{table_path}; std::map restored_context_builder_options; WriteContextBuilder restored_context_builder("restored-writer"); restored_context_builder_options = options; @@ -891,7 +891,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) { restored_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto restored_write, - FileStoreWrite::Create(restored_context_builder_load_options, std::move(restored_context), + FileStoreWrite::Create(restored_context_builder_load_context, std::move(restored_context), restored_context_builder_options)); auto key_value_write = dynamic_cast(restored_write.get()); ASSERT_NE(nullptr, key_value_write); @@ -921,14 +921,14 @@ TEST_F(KeyValueFileStoreWriteTest, TestSpillDiskQuotaExhaustedFallsBackToFlushDa {Options::WRITE_BUFFER_SPILL_MAX_DISK_SIZE, "1b"}}, /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("test"); context_builder.WithTempDirectory(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); // Disk quota is 1 byte, so spill will exhaust quota immediately and fall back to @@ -973,14 +973,14 @@ TEST_F(KeyValueFileStoreWriteTest, TestMultiRoundSpillWithSameKeyDeduplication) {Options::WRITE_BUFFER_SPILLABLE, "true"}}, /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("test"); context_builder.WithTempDirectory(dir->Str()).WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); // Round 1: alice, bob, alice (duplicate key) → after dedup: alice + bob = 2 rows diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp index 850eff66b..86b334acb 100644 --- a/src/paimon/core/operation/merge_file_split_read_test.cpp +++ b/src/paimon/core/operation/merge_file_split_read_test.cpp @@ -31,7 +31,7 @@ #include "arrow/c/abi.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/reader/concat_batch_reader.h" @@ -93,17 +93,17 @@ class MergeFileSplitReadTest : public ::testing::Test, std::shared_ptr CreateInternalReadContext( const std::shared_ptr& read_context, - const TableLoadOptions& read_context_load_options, + const TableLoadContext& read_context_load_context, const std::map& read_context_options, int32_t schema_id = 0) { - SchemaManager schema_manager(fs_, read_context_load_options.path); + SchemaManager schema_manager(fs_, read_context_load_context.path); EXPECT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(schema_id)); EXPECT_OK_AND_ASSIGN( auto context, InternalReadContext::Create( read_context, table_schema, read_context_options, - read_context_load_options.file_system, read_context_load_options.path, - read_context_load_options.fs_scheme_to_identifier_map, - read_context_load_options.catalog, - read_context_load_options.identifier, read_context_options)); + read_context_load_context.file_system, read_context_load_context.path, + read_context_load_context.fs_scheme_to_identifier_map, + read_context_load_context.catalog, + read_context_load_context.identifier, read_context_options)); return context; } @@ -618,8 +618,8 @@ TEST_F(MergeFileSplitReadTest, TestGenerateKeyPredicates2) { TEST_P(MergeFileSplitReadTest, TestSimple) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -637,12 +637,12 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { {Options::IGNORE_DELETE, "true"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN( std::unique_ptr split_read, @@ -691,8 +691,8 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(0, arrow::field("k0", arrow::int32())), @@ -708,11 +708,11 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) {Options::MERGE_ENGINE, "deduplicate"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); std::shared_ptr internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr split_read, CreateMergeFileSplitRead(internal_context)); @@ -765,8 +765,8 @@ TEST_P(MergeFileSplitReadTest, TestRealtimeReadConcatenatesOrderedDiskSections) TEST_P(MergeFileSplitReadTest, TestLookUp) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -785,12 +785,12 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { {Options::FORCE_LOOKUP, "true"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -820,8 +820,8 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options = {{Options::SEQUENCE_FIELD, "s0,s1"}, @@ -830,12 +830,12 @@ TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { {Options::READ_BATCH_SIZE, "1"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); @@ -853,8 +853,8 @@ TEST_P(MergeFileSplitReadTest, TestReadWithLimits) { TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -870,12 +870,12 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { context_builder_options = {{Options::MERGE_ENGINE, "deduplicate"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -900,8 +900,8 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -936,12 +936,12 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate_result, PredicateBuilder::And({less_than, greater_or_equal, greater_than})); context_builder.SetPredicate(predicate_result); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -970,8 +970,8 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -1001,12 +1001,12 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { PredicateBuilder::And({greater_or_equal, greater_than})); context_builder.SetPredicate(predicate_result); context_builder.EnablePredicateFilter(true).EnableLateMaterializing(true); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1034,8 +1034,8 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicateAndLateMaterializing) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -1058,11 +1058,11 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, read_context_load_options, + auto internal_context = CreateInternalReadContext(read_context, read_context_load_context, read_context_options, /*schema_id=*/7); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); @@ -1093,8 +1093,8 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -1114,11 +1114,11 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); - auto internal_context = CreateInternalReadContext(read_context, read_context_load_options, + auto internal_context = CreateInternalReadContext(read_context, read_context_load_context, read_context_options, /*schema_id=*/7); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); @@ -1149,8 +1149,8 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -1170,11 +1170,11 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { {Options::IGNORE_DELETE, "true"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1204,8 +1204,8 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_with_mor.db/pk_table_with_mor"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -1224,11 +1224,11 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { {Options::IGNORE_DELETE, "true"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1258,8 +1258,8 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -1276,11 +1276,11 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { {Options::IGNORE_DELETE, "true"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1305,8 +1305,8 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDelete) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -1323,11 +1323,11 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, PrepareDataSplit2())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, @@ -1352,8 +1352,8 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { std::string path = paimon::test::GetDataDir() + "/parquet/pk_table_partial_update.db/pk_table_partial_update"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; @@ -1370,11 +1370,11 @@ TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); std::vector> empty_data_split; ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, empty_data_split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, @@ -1392,8 +1392,8 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { DataField(6, arrow::field("v0", arrow::float64())), DataField(7, arrow::field("v1", arrow::boolean()))}; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); @@ -1405,12 +1405,12 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { {Options::IGNORE_DELETE, "true"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); auto fields_with_row_kind = read_schema->fields(); fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -1450,8 +1450,8 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { std::string path = paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; std::vector raw_read_fields = {DataField(3, arrow::field("f3", arrow::float64())), @@ -1466,7 +1466,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { {"orc.read.enable-metrics", "true"}}; AddOptions(&context_builder, context_builder_options); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); auto meta1 = std::make_shared( @@ -1512,7 +1512,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { ASSERT_OK_AND_ASSIGN(auto data_split, builder.WithSnapshot(5).IsStreaming(false).RawConvertible(false).Build()); auto internal_context = - CreateInternalReadContext(read_context, read_context_load_options, read_context_options); + CreateInternalReadContext(read_context, read_context_load_context, read_context_options); ASSERT_OK_AND_ASSIGN(auto batch_reader, CreateReader(internal_context, {data_split})); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, ReadResultCollector::CollectResult(batch_reader.get())); diff --git a/src/paimon/core/operation/precise_file_index_read_test.cpp b/src/paimon/core/operation/precise_file_index_read_test.cpp index faf139558..76daf8180 100644 --- a/src/paimon/core/operation/precise_file_index_read_test.cpp +++ b/src/paimon/core/operation/precise_file_index_read_test.cpp @@ -30,7 +30,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/file_index/bitmap/bitmap_file_index.h" #include "paimon/common/predicate/leaf_predicate_impl.h" @@ -146,12 +146,12 @@ class PreciseFileIndexReadTest : public ::testing::Test { Result> CreateSplitRead( const std::shared_ptr& read_context, - const TableLoadOptions& read_context_load_options, + const TableLoadContext& read_context_load_context, const std::map& read_context_options, std::shared_ptr table_schema = nullptr) const { if (!table_schema) { SchemaManager schema_manager(std::make_shared(), - read_context_load_options.path); + read_context_load_context.path); PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(0)); } auto options = table_schema->Options(); @@ -161,17 +161,17 @@ class PreciseFileIndexReadTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr context, InternalReadContext::Create( - read_context, table_schema, options, read_context_load_options.file_system, - read_context_load_options.path, - read_context_load_options.fs_scheme_to_identifier_map, - read_context_load_options.catalog, read_context_load_options.identifier, + read_context, table_schema, options, read_context_load_context.file_system, + read_context_load_context.path, + read_context_load_context.fs_scheme_to_identifier_map, + read_context_load_context.catalog, read_context_load_context.identifier, read_context_options)); return std::make_unique(nullptr, context, pool_, nullptr); } Result> ReadWithFileIndex( const std::shared_ptr& read_context, - const TableLoadOptions& read_context_load_options, + const TableLoadContext& read_context_load_context, const std::map& read_context_options, const std::shared_ptr& index, const RoaringBitmap32& deleted_rows = RoaringBitmap32(), @@ -180,7 +180,7 @@ class PreciseFileIndexReadTest : public ::testing::Test { std::shared_ptr deletion_vector = nullptr, const std::optional& row_count = std::nullopt) const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_read, - CreateSplitRead(read_context, read_context_load_options, + CreateSplitRead(read_context, read_context_load_context, read_context_options, std::move(table_schema))); auto file = PrepareDataFile(); file->embedded_index = index; @@ -220,16 +220,16 @@ class PreciseFileIndexReadTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr index, MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, matches, pool_)); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.SetReadFieldNames({"f0"}).SetPredicate(predicate).EnablePreciseFileIndexEvaluate( true); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, builder.Finish()); - return ReadWithFileIndex(context, context_load_options, context_options, index); + return ReadWithFileIndex(context, context_load_context, context_options, index); } static void CheckSearchResult(const std::shared_ptr& actual, @@ -276,18 +276,18 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateNestedAndOrWithPartitionConst "f0", statistics_index_type_, {{first_query, {{{0, 0.2f}, {1, 0.4f}}, 0.2}}, {second_query, {{{2, 0.6f}}, 0.1}}}, pool_)); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder_options[Options::READ_BATCH_SIZE] = "1"; builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}).SetPredicate(predicate); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, context_load_options, context_options, index)); + ReadWithFileIndex(context, context_load_context, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}, @@ -320,17 +320,17 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateReordersQueries) { {{wide_query, {{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, 0.9}}, {narrow_query, {{{0, 0.4f}, {2, 0.5f}}, 0.1}}}, pool_)); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0", "f3"}).SetPredicate(predicate); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, context_load_options, context_options, index)); + ReadWithFileIndex(context, context_load_context, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, R"([["Tony",14.1]])"); @@ -476,20 +476,20 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateEmptyRowsShortCircuitExecutio {{empty_query, {{}, std::nullopt}}, {all_query, {{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, std::nullopt}}}, pool_)); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0"}).SetPredicate(predicate); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); for (bool all_deleted : {false, true}) { statistics_->executions.clear(); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadWithFileIndex(context, context_load_options, context_options, index, + ReadWithFileIndex(context, context_load_context, context_options, index, all_deleted ? RoaringBitmap32::From({0, 1, 2}) : RoaringBitmap32())); ASSERT_TRUE(!result || result->length() == 0); ASSERT_EQ(statistics_->executions.size(), all_deleted ? 0 : 1); @@ -539,19 +539,19 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateIndexedLeafSkipsUnselectedSca std::shared_ptr index, MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, {{query, {{{0, 0.25f}, {2, 0.75f}}, 0.1}}}, pool_)); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); // Single-row batches let bitmap selection skip the unselected scalar evaluation batch. builder_options[Options::READ_BATCH_SIZE] = "1"; builder.SetReadFieldNames({"f0", "f3"}).SetPredicate(predicate); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, context_load_options, context_options, index)); + ReadWithFileIndex(context, context_load_context, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, R"([["Tony",14.1]])"); @@ -588,18 +588,18 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRechecksScalarIndexCandidates std::shared_ptr predicate, PredicateBuilder::And( {PredicateBuilder::Equal(1, "f3", FieldType::DOUBLE, Literal(13.1)), text})); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder_options[Options::READ_BATCH_SIZE] = "1"; builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}).SetPredicate(predicate); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, context_load_options, context_options, index)); + ReadWithFileIndex(context, context_load_context, context_options, index)); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), field, arrow::field("_f0_INDEX_SCORE", arrow::float32())}, @@ -618,19 +618,19 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicatePrechecksSearchBeforeEmptyCan PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(case_id == 0 ? 20 : 10)); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({scalar, text})); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); const auto deleted = case_id == 1 ? RoaringBitmap32::From({0, 1, 2}) : RoaringBitmap32(); const std::optional> ranges = case_id == 2 ? std::make_optional(std::vector()) : std::nullopt; - ASSERT_NOK_WITH_MSG(ReadWithFileIndex(context, context_load_options, context_options, + ASSERT_NOK_WITH_MSG(ReadWithFileIndex(context, context_load_context, context_options, nullptr, deleted, nullptr, ranges), "has no File Index for search field 'f0'"); } @@ -660,8 +660,8 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRenamedTextAndMissingScalarCo MockSearchFileIndex::MakeQueryIndex( "f0", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, 0.5}}}, pool_)); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); @@ -670,11 +670,11 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRenamedTextAndMissingScalarCo .SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) .SetPredicate(predicate); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, context_load_options, context_options, index, + ReadWithFileIndex(context, context_load_context, context_options, index, RoaringBitmap32(), current)); CheckSearchResult( result, @@ -686,8 +686,8 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRenamedTextAndMissingScalarCo ASSERT_OK_AND_ASSIGN( std::shared_ptr nonnull, PredicateBuilder::And({text, PredicateBuilder::IsNotNull(1, "new_value", FieldType::INT)})); - TableLoadOptions nonnull_builder_load_options; - nonnull_builder_load_options.path = SearchTablePath(); + TableLoadContext nonnull_builder_load_context; + nonnull_builder_load_context.path = SearchTablePath(); std::map nonnull_builder_options; ReadContextBuilder nonnull_builder; nonnull_builder.EnablePreciseFileIndexEvaluate(true); @@ -696,12 +696,12 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRenamedTextAndMissingScalarCo .SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) .SetPredicate(nonnull); - auto nonnull_context_load_options = nonnull_builder_load_options; + auto nonnull_context_load_context = nonnull_builder_load_context; auto nonnull_context_options = nonnull_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr nonnull_context, nonnull_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr empty, - ReadWithFileIndex(nonnull_context, nonnull_context_load_options, nonnull_context_options, + ReadWithFileIndex(nonnull_context, nonnull_context_load_context, nonnull_context_options, index, RoaringBitmap32(), current)); ASSERT_TRUE(!empty || empty->length() == 0); } @@ -729,8 +729,8 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateFullTextAndVectorScoresAfterF auto search = std::make_shared("f3", 1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); @@ -738,7 +738,7 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateFullTextAndVectorScoresAfterF builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0", "f3", "_f0_INDEX_SCORE"}) .SetPredicate(predicate) .SetVectorSearch(search); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); for (int32_t case_id : {0, 1, 2}) { @@ -746,7 +746,7 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateFullTextAndVectorScoresAfterF const std::optional> ranges = case_id == 2 ? std::make_optional(std::vector{Range(2, 2)}) : std::nullopt; ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - ReadWithFileIndex(context, context_load_options, context_options, + ReadWithFileIndex(context, context_load_context, context_options, index, deleted, nullptr, ranges)); CheckSearchResult( result, @@ -760,8 +760,8 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateFullTextAndVectorScoresAfterF TEST_F(PreciseFileIndexReadTest, TestFullTextRootScoresAndLimitAfterScalarDvAndRanges) { const std::string query = R"({"match":{"query":"ranked"}})"; auto search = std::make_shared("f0", query, 1, RoaringBitmap64::From({0, 2})); - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); @@ -769,7 +769,7 @@ TEST_F(PreciseFileIndexReadTest, TestFullTextRootScoresAndLimitAfterScalarDvAndR builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}) .SetFullTextSearch(search) .SetPredicate(PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.5))); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( @@ -779,7 +779,7 @@ TEST_F(PreciseFileIndexReadTest, TestFullTextRootScoresAndLimitAfterScalarDvAndR {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, std::nullopt}}}, pool_)); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadWithFileIndex(context, context_load_options, context_options, index, + ReadWithFileIndex(context, context_load_context, context_options, index, RoaringBitmap32::From({0}), nullptr, std::vector{Range(1, 2)})); CheckSearchResult(result, {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64()), @@ -804,8 +804,8 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec MockSearchFileIndex::Serialize(indexes, pool_)); for (const std::string& query_type : std::vector{"none", "scalar", "match", "full-text", "vector"}) { - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.SetReadFieldNames({"f0", "f3"}).EnablePreciseFileIndexEvaluate(true); @@ -824,7 +824,7 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{})); } - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); for (bool empty_dv : {false, true}) { @@ -836,7 +836,7 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec const std::optional> ranges = empty_candidates ? std::make_optional(std::vector()) : std::nullopt; ASSERT_NOK_WITH_MSG( - ReadWithFileIndex(context, context_load_options, context_options, index, + ReadWithFileIndex(context, context_load_context, context_options, index, RoaringBitmap32(), nullptr, ranges, dv), "Precise file-index evaluation requires bitmap32 deletion vectors"); } @@ -847,8 +847,8 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVec TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Limit) { for (const std::string& search_type : std::vector{"none", "scalar", "match", "full-text", "vector"}) { - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); @@ -869,11 +869,11 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Li "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{})); } - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_NOK_WITH_MSG( - ReadWithFileIndex(context, context_load_options, context_options, /*index=*/nullptr, + ReadWithFileIndex(context, context_load_context, context_options, /*index=*/nullptr, RoaringBitmap32(), nullptr, /*row_ranges=*/std::nullopt, /*deletion_vector=*/nullptr, int64_t{RoaringBitmap32::MAX_VALUE} + 1), @@ -883,8 +883,8 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Li TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationAtBitmap32Limit) { for (bool has_predicate : {false, true}) { - TableLoadOptions builder_load_options; - builder_load_options.path = SearchTablePath(); + TableLoadContext builder_load_context; + builder_load_context.path = SearchTablePath(); std::map builder_options; ReadContextBuilder builder; builder.SetReadFieldNames({"f0", "f3"}).EnablePreciseFileIndexEvaluate(true); @@ -892,12 +892,12 @@ TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationAtBitmap32Limit) { builder.SetPredicate( PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0))); } - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadWithFileIndex(context, context_load_options, context_options, /*index=*/nullptr, + ReadWithFileIndex(context, context_load_context, context_options, /*index=*/nullptr, RoaringBitmap32(), nullptr, std::vector{Range(0, 2)}, /*deletion_vector=*/nullptr, RoaringBitmap32::MAX_VALUE)); CheckSearchResult(result, diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 43da854fe..f56ad516f 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -29,7 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/reader/concat_batch_reader.h" #include "paimon/common/types/data_field.h" @@ -200,12 +200,12 @@ class RawFileSplitReadTest : public ::testing::Test { Result> ReadSearchWithDeletionVector( const std::shared_ptr& read_context, - const TableLoadOptions& read_context_load_options, + const TableLoadContext& read_context_load_context, const std::map& read_context_options, const std::string& index_type, const RoaringBitmap32& deleted_rows, std::shared_ptr table_schema) const { SchemaManager schema_manager(std::make_shared(), - read_context_load_options.path); + read_context_load_context.path); if (!table_schema) { PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(0)); } @@ -213,9 +213,9 @@ class RawFileSplitReadTest : public ::testing::Test { std::unique_ptr internal_context, InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - read_context_load_options.file_system, read_context_load_options.path, - read_context_load_options.fs_scheme_to_identifier_map, - read_context_load_options.catalog, read_context_load_options.identifier, + read_context_load_context.file_system, read_context_load_context.path, + read_context_load_context.fs_scheme_to_identifier_map, + read_context_load_context.catalog, read_context_load_context.identifier, read_context_options)); auto split = std::dynamic_pointer_cast(PrepareDataSplits().front()); if (!split) { @@ -235,7 +235,7 @@ class RawFileSplitReadTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( - read_context_load_options.path, schema, table_schema->PartitionKeys(), + read_context_load_context.path, schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, options.IndexFileInDataFileDir(), @@ -262,8 +262,8 @@ class RawFileSplitReadTest : public ::testing::Test { std::string path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/" "multi_partition_append_table"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames(read_schema->field_names()); @@ -272,20 +272,20 @@ class RawFileSplitReadTest : public ::testing::Test { context_builder.SetPredicate(PredicateBuilder::IsNotNull(0, "f0", FieldType::STRING)); } - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - read_context_load_options.path); + read_context_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( auto internal_context, InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - read_context_load_options.file_system, read_context_load_options.path, - read_context_load_options.fs_scheme_to_identifier_map, - read_context_load_options.catalog, read_context_load_options.identifier, + read_context_load_context.file_system, read_context_load_context.path, + read_context_load_context.fs_scheme_to_identifier_map, + read_context_load_context.catalog, read_context_load_context.identifier, read_context_options)); auto data_splits = PrepareDataSplits(); const auto& core_options = internal_context->GetCoreOptions(); @@ -365,14 +365,14 @@ TEST_F(RawFileSplitReadTest, TestCreateReader) { TEST_F(RawFileSplitReadTest, TestCreateReaderUsesExplicitPredicate) { const std::string path = GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; - TableLoadOptions builder_load_options; - builder_load_options.path = path; + TableLoadContext builder_load_context; + builder_load_context.path = path; std::map builder_options; ReadContextBuilder builder; builder_options[Options::FILE_INDEX_READ_ENABLED] = "false"; builder.EnablePredicateFilter(true).SetPredicate( PredicateBuilder::Equal(0, "f0", FieldType::STRING, Literal(FieldType::STRING, "Bob", 3))); - auto read_context_load_options = builder_load_options; + auto read_context_load_context = builder_load_context; auto read_context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); SchemaManager schemas(std::make_shared(), path); @@ -380,10 +380,10 @@ TEST_F(RawFileSplitReadTest, TestCreateReaderUsesExplicitPredicate) { ASSERT_OK_AND_ASSIGN(std::shared_ptr context, InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - read_context_load_options.file_system, read_context_load_options.path, - read_context_load_options.fs_scheme_to_identifier_map, - read_context_load_options.catalog, - read_context_load_options.identifier, read_context_options)); + read_context_load_context.file_system, read_context_load_context.path, + read_context_load_context.fs_scheme_to_identifier_map, + read_context_load_context.catalog, + read_context_load_context.identifier, read_context_options)); const CoreOptions& options = context->GetCoreOptions(); const auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, options.CreateExternalPaths()); @@ -636,15 +636,15 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { std::shared_ptr vector_search = std::make_shared( "f3", /*limit=*/2, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true); context_builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}) .SetVectorSearch(vector_search) .WithCache(cache); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), path); @@ -652,10 +652,10 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - read_context_load_options.file_system, read_context_load_options.path, - read_context_load_options.fs_scheme_to_identifier_map, - read_context_load_options.catalog, - read_context_load_options.identifier, read_context_options)); + read_context_load_context.file_system, read_context_load_context.path, + read_context_load_context.fs_scheme_to_identifier_map, + read_context_load_context.catalog, + read_context_load_context.identifier, read_context_options)); auto data_splits = PrepareDataSplits(); auto data_split = std::dynamic_pointer_cast(data_splits.front()); @@ -734,13 +734,13 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { ASSERT_EQ(cache->GetCount(CacheKind::INDEX_READER), 4); ASSERT_EQ(cache->SupplierCallCount(CacheKind::INDEX_READER), 0); - TableLoadOptions without_score_builder_load_options; - without_score_builder_load_options.path = path; + TableLoadContext without_score_builder_load_context; + without_score_builder_load_context.path = path; std::map without_score_builder_options; ReadContextBuilder without_score_builder; without_score_builder.EnablePreciseFileIndexEvaluate(true); without_score_builder.SetReadFieldNames({"f0"}).SetVectorSearch(vector_search); - auto without_score_context_load_options = without_score_builder_load_options; + auto without_score_context_load_context = without_score_builder_load_context; auto without_score_context_options = without_score_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr without_score_context, without_score_builder.Finish()); @@ -748,10 +748,10 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { auto without_score_internal_context, InternalReadContext::Create( without_score_context, table_schema, table_schema->Options(), - without_score_context_load_options.file_system, without_score_context_load_options.path, - without_score_context_load_options.fs_scheme_to_identifier_map, - without_score_context_load_options.catalog, - without_score_context_load_options.identifier, without_score_context_options)); + without_score_context_load_context.file_system, without_score_context_load_context.path, + without_score_context_load_context.fs_scheme_to_identifier_map, + without_score_context_load_context.catalog, + without_score_context_load_context.identifier, without_score_context_options)); auto without_score_split_read = std::make_unique( path_factory, std::move(without_score_internal_context), pool_, executor); ASSERT_OK_AND_ASSIGN(std::unique_ptr without_score_reader, @@ -776,18 +776,18 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchFiltersDeletionVectorBeforeTopK) { std::shared_ptr search = std::make_shared( "f3", /*limit=*/1, std::vector{1.0f}, user_filter, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - TableLoadOptions builder_load_options; - builder_load_options.path = path; + TableLoadContext builder_load_context; + builder_load_context.path = path; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}).SetVectorSearch(search); - auto read_context_load_options = builder_load_options; + auto read_context_load_context = builder_load_context; auto read_context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadSearchWithDeletionVector(read_context, read_context_load_options, read_context_options, + ReadSearchWithDeletionVector(read_context, read_context_load_context, read_context_options, MockSearchFileIndex::VECTOR_IDENTIFIER, RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); std::shared_ptr expected; @@ -808,18 +808,18 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchFiltersDeletionVectorBeforeLimit) user_filter.Add(2); auto search = std::make_shared("f3", R"({"match":{"query":"match"}})", /*limit=*/1, user_filter); - TableLoadOptions builder_load_options; - builder_load_options.path = path; + TableLoadContext builder_load_context; + builder_load_context.path = path; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); - auto read_context_load_options = builder_load_options; + auto read_context_load_context = builder_load_context; auto read_context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, - ReadSearchWithDeletionVector(read_context, read_context_load_options, read_context_options, + ReadSearchWithDeletionVector(read_context, read_context_load_context, read_context_options, MockSearchFileIndex::FULL_TEXT_IDENTIFIER, RoaringBitmap32::From({0}), /*table_schema=*/nullptr)); std::shared_ptr expected; @@ -860,14 +860,14 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { auto search = std::make_shared("renamed_f3", 2, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - TableLoadOptions builder_load_options; - builder_load_options.path = path; + TableLoadContext builder_load_context; + builder_load_context.path = path; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder_options = options; builder.SetReadFieldNames({"f0", "_renamed_f3_INDEX_SCORE"}).SetVectorSearch(search); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); @@ -880,7 +880,7 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector( - context, context_load_options, context_options, + context, context_load_context, context_options, identifier, deleted, current_schema)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); @@ -897,18 +897,18 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; - TableLoadOptions missing_field_builder_load_options; - missing_field_builder_load_options.path = path; + TableLoadContext missing_field_builder_load_context; + missing_field_builder_load_context.path = path; std::map missing_field_builder_options; ReadContextBuilder missing_field_builder; missing_field_builder.EnablePreciseFileIndexEvaluate(true); missing_field_builder_options = options; missing_field_builder.SetReadFieldNames({"f0"}).SetVectorSearch(missing_search); - auto missing_context_load_options = missing_field_builder_load_options; + auto missing_context_load_context = missing_field_builder_load_context; auto missing_context_options = missing_field_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); - ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, missing_context_load_options, + ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, missing_context_load_context, missing_context_options, identifier, RoaringBitmap32(), current_schema), "has no matching field ID"); @@ -942,14 +942,14 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) original_schema->PartitionKeys(), {}, options)); auto search = std::make_shared("renamed_f3", R"({"match":{"query":"match"}})", 2); - TableLoadOptions builder_load_options; - builder_load_options.path = path; + TableLoadContext builder_load_context; + builder_load_context.path = path; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true); builder_options = options; builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); @@ -961,7 +961,7 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) for (const auto& [deleted, expected_json] : cases) { SCOPED_TRACE(deleted.ToString()); ASSERT_OK_AND_ASSIGN(auto actual, ReadSearchWithDeletionVector( - context, context_load_options, context_options, + context, context_load_context, context_options, identifier, deleted, current_schema)); auto expected_result = arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json); @@ -978,18 +978,18 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) // The old file's f3 is field ID 3; the current table's f3 is a different field with ID 4. auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; - TableLoadOptions missing_field_builder_load_options; - missing_field_builder_load_options.path = path; + TableLoadContext missing_field_builder_load_context; + missing_field_builder_load_context.path = path; std::map missing_field_builder_options; ReadContextBuilder missing_field_builder; missing_field_builder.EnablePreciseFileIndexEvaluate(true); missing_field_builder_options = options; missing_field_builder.SetReadFieldNames({"f0"}).SetFullTextSearch(missing_search); - auto missing_context_load_options = missing_field_builder_load_options; + auto missing_context_load_context = missing_field_builder_load_context; auto missing_context_options = missing_field_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, missing_field_builder.Finish()); - ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, missing_context_load_options, + ASSERT_NOK_WITH_MSG(ReadSearchWithDeletionVector(missing_context, missing_context_load_context, missing_context_options, identifier, RoaringBitmap32(), current_schema), "has no matching field ID"); @@ -999,24 +999,24 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { std::string path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/" "multi_partition_append_table"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - read_context_load_options.path); + read_context_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(auto internal_context, InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - read_context_load_options.file_system, read_context_load_options.path, - read_context_load_options.fs_scheme_to_identifier_map, - read_context_load_options.catalog, - read_context_load_options.identifier, read_context_options)); + read_context_load_context.file_system, read_context_load_context.path, + read_context_load_context.fs_scheme_to_identifier_map, + read_context_load_context.catalog, + read_context_load_context.identifier, read_context_options)); const auto& core_options = internal_context->GetCoreOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, @@ -1059,24 +1059,24 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { TEST_F(RawFileSplitReadTest, TestMatch) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = path; + TableLoadContext context_builder_load_context; + context_builder_load_context.path = path; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - auto read_context_load_options = context_builder_load_options; + auto read_context_load_context = context_builder_load_context; auto read_context_options = context_builder_options; ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), - read_context_load_options.path); + read_context_load_context.path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN(std::shared_ptr internal_context, InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - read_context_load_options.file_system, read_context_load_options.path, - read_context_load_options.fs_scheme_to_identifier_map, - read_context_load_options.catalog, - read_context_load_options.identifier, read_context_options)); + read_context_load_context.file_system, read_context_load_context.path, + read_context_load_context.fs_scheme_to_identifier_map, + read_context_load_context.catalog, + read_context_load_context.identifier, read_context_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique( diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 2b5f53e57..2593e58cb 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -24,7 +24,7 @@ #include "arrow/c/bridge.h" #include "arrow/type.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/catalog_options.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/core/utils/branch_manager.h" diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 343917eff..99f45f87f 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -19,7 +19,7 @@ #include "paimon/scan_context.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/defs.h" #include "paimon/executor.h" diff --git a/src/paimon/core/operation/write_context_test.cpp b/src/paimon/core/operation/write_context_test.cpp index de15926e1..3db5f8201 100644 --- a/src/paimon/core/operation/write_context_test.cpp +++ b/src/paimon/core/operation/write_context_test.cpp @@ -21,7 +21,7 @@ #include #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" diff --git a/src/paimon/core/snapshot_file_scan_test.cpp b/src/paimon/core/snapshot_file_scan_test.cpp index e9facf193..1ce96b5d3 100644 --- a/src/paimon/core/snapshot_file_scan_test.cpp +++ b/src/paimon/core/snapshot_file_scan_test.cpp @@ -32,7 +32,7 @@ #include #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/utils/path_util.h" @@ -466,7 +466,7 @@ TEST(SnapshotFileScanTest, TestCommittedRealtimeOffsetFiles) { const std::string table_path = dir->Str() + "/foo.db/bar"; ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("test"); write_builder_options[Options::REALTIME_ENABLED] = "true"; @@ -474,7 +474,7 @@ TEST(SnapshotFileScanTest, TestCommittedRealtimeOffsetFiles) { std::unique_ptr write_context, write_builder.WithStreamingMode(true).WithRealtimeContext(realtime_context).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); auto input_type = arrow::struct_({arrow::field("_REALTIME_OFFSET", arrow::int64(), false), schema->field(0), schema->field(1)}); @@ -484,14 +484,14 @@ TEST(SnapshotFileScanTest, TestCommittedRealtimeOffsetFiles) { ASSERT_OK(writer->Write(std::move(batch))); ASSERT_OK_AND_ASSIGN(auto progress, writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); ASSERT_FALSE(progress.empty()); - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("test"); commit_builder_options[Options::REALTIME_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, commit->CommitWithProgress(progress, 0, std::nullopt)); diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index bba59fcb6..307ed2187 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -35,7 +35,7 @@ #include "gtest/gtest.h" #include "paimon/cache/cache.h" #include "paimon/catalog/table.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/table/special_fields.h" @@ -659,7 +659,7 @@ TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { SCOPED_TRACE(vector_search); SCOPED_TRACE(entrypoint); ReadContextBuilder builder; - TableLoadOptions context_load_options{dir->Str()}; + TableLoadContext context_load_context{dir->Str()}; std::map context_options; builder.EnablePreciseFileIndexEvaluate(true); if (vector_search) { @@ -676,7 +676,7 @@ TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { loaded_table->Copy(context_options)); return configured->NewRead(std::move(context)); })() - : TableRead::Create(context_load_options, + : TableRead::Create(context_load_context, std::move(context), context_options); ASSERT_NOK_WITH_MSG(read, "Precise file-index evaluation is not supported for format tables"); @@ -821,43 +821,43 @@ TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { const std::map refused = { {Options::METASTORE_PARTITIONED_TABLE, "true"}}; - TableLoadOptions scan_builder_load_options{dir->Str()}; + TableLoadContext scan_builder_load_context{dir->Str()}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options = refused; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options), "metastore.partitioned-table"); - TableLoadOptions read_builder_load_options{dir->Str()}; + TableLoadContext read_builder_load_context{dir->Str()}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = refused; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(read_builder_load_options, std::move(read_context), read_builder_options), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options), "metastore.partitioned-table"); - TableLoadOptions write_builder_load_options{dir->Str()}; + TableLoadContext write_builder_load_context{dir->Str()}; std::map write_builder_options; WriteContextBuilder write_builder("test-user"); write_builder_options = refused; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options), "metastore.partitioned-table"); - TableLoadOptions commit_builder_load_options{dir->Str()}; + TableLoadContext commit_builder_load_context{dir->Str()}; std::map commit_builder_options; CommitContextBuilder commit_builder("test-user"); commit_builder_options = refused; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(commit_builder_load_options, + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options), "metastore.partitioned-table"); } @@ -886,7 +886,7 @@ TEST(FormatTableTest, TestQueryAuthInTheSchemaCannotBeDisabledAtTheCall) { ASSERT_TRUE(created.IsInvalid()) << created.ToString(); ASSERT_NOK_WITH_MSG(created, refused); - TableLoadOptions scan_builder_load{dir->Str()}; + TableLoadContext scan_builder_load{dir->Str()}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options = disabling; @@ -897,7 +897,7 @@ TEST(FormatTableTest, TestQueryAuthInTheSchemaCannotBeDisabledAtTheCall) { ASSERT_TRUE(scanned.IsInvalid()) << scanned.ToString(); ASSERT_NOK_WITH_MSG(scanned, refused); - TableLoadOptions read_builder_load{dir->Str()}; + TableLoadContext read_builder_load{dir->Str()}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = disabling; @@ -1181,7 +1181,7 @@ TEST(FormatTableTest, TestCatalogFactoriesRetainExternalLocationWithExplicitPath /*location_carries_paimon_metadata=*/false, {})); auto catalog = std::make_shared(); catalog->SetTable(external); - TableLoadOptions load; + TableLoadContext load; load.catalog = catalog; load.identifier.emplace("db", "tbl"); if (explicit_path) { @@ -2877,12 +2877,12 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - TableLoadOptions write_builder_load_options{dir->Str()}; + TableLoadContext write_builder_load_context{dir->Str()}; std::map write_builder_options; WriteContextBuilder write_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); ASSERT_NOK_WITH_MSG(write->GetBaseSnapshotId({{"dt", "20240101"}}, 0), "format tables do not have base snapshots"); @@ -2899,81 +2899,81 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { "cannot be compacted"); // A write id prefixes a postpone-bucket writer's files; a format table has no buckets. - TableLoadOptions write_id_builder_load_options{dir->Str()}; + TableLoadContext write_id_builder_load_context{dir->Str()}; std::map write_id_builder_options; WriteContextBuilder write_id_builder("test-user"); write_id_builder.WithWriteId(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_context, write_id_builder.Finish()); ASSERT_NOK_WITH_MSG( - FileStoreWrite::Create(write_id_builder_load_options, std::move(write_id_context), + FileStoreWrite::Create(write_id_builder_load_context, std::move(write_id_context), write_id_builder_options), "a write id would name nothing"); - TableLoadOptions empty_commit_builder_load_options{dir->Str()}; + TableLoadContext empty_commit_builder_load_context{dir->Str()}; std::map empty_commit_builder_options; CommitContextBuilder empty_commit_builder("test-user"); empty_commit_builder.IgnoreEmptyCommit(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_commit_context, empty_commit_builder.Finish()); ASSERT_NOK_WITH_MSG( - FileStoreCommit::Create(empty_commit_builder_load_options, std::move(empty_commit_context), + FileStoreCommit::Create(empty_commit_builder_load_context, std::move(empty_commit_context), empty_commit_builder_options), "cannot record an empty commit"); - TableLoadOptions rest_builder_load_options{dir->Str()}; + TableLoadContext rest_builder_load_context{dir->Str()}; std::map rest_builder_options; CommitContextBuilder rest_builder("test-user"); rest_builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, rest_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(rest_builder_load_options, std::move(rest_context), + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(rest_builder_load_context, std::move(rest_context), rest_builder_options), "rest catalog"); - TableLoadOptions table_id_builder_load_options{dir->Str()}; + TableLoadContext table_id_builder_load_context{dir->Str()}; std::map table_id_builder_options; CommitContextBuilder table_id_builder("test-user"); - table_id_builder_load_options.table_id = "table-uuid"; + table_id_builder_load_context.table_id = "table-uuid"; ASSERT_OK_AND_ASSIGN(std::unique_ptr table_id_context, table_id_builder.Finish()); ASSERT_NOK_WITH_MSG( - FileStoreCommit::Create(table_id_builder_load_options, std::move(table_id_context), + FileStoreCommit::Create(table_id_builder_load_context, std::move(table_id_context), table_id_builder_options), "a table id requires a catalog or an external REST commit request"); auto format_catalog = std::make_shared(); format_catalog->SetTable(table); - TableLoadOptions catalog_builder_load_options{dir->Str()}; + TableLoadContext catalog_builder_load_context{dir->Str()}; std::map catalog_builder_options; CommitContextBuilder catalog_builder("test-user"); - catalog_builder_load_options.catalog = format_catalog; - catalog_builder_load_options.identifier.emplace(Identifier("db", "fmt")); + catalog_builder_load_context.catalog = format_catalog; + catalog_builder_load_context.identifier.emplace(Identifier("db", "fmt")); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); ASSERT_OK_AND_ASSIGN( [[maybe_unused]] std::unique_ptr catalog_commit, - FileStoreCommit::Create(catalog_builder_load_options, std::move(catalog_context), + FileStoreCommit::Create(catalog_builder_load_context, std::move(catalog_context), catalog_builder_options)); ASSERT_TRUE(format_catalog->CommitCalls().empty()); - TableLoadOptions conflict_builder_load_options{dir->Str()}; + TableLoadContext conflict_builder_load_context{dir->Str()}; std::map conflict_builder_options; CommitContextBuilder conflict_builder("test-user"); conflict_builder.AppendCommitCheckConflict(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr conflict_context, conflict_builder.Finish()); ASSERT_NOK_WITH_MSG( - FileStoreCommit::Create(conflict_builder_load_options, std::move(conflict_context), + FileStoreCommit::Create(conflict_builder_load_context, std::move(conflict_context), conflict_builder_options), "no manifests to check a concurrent commit against"); - TableLoadOptions commit_builder_load_options{dir->Str()}; + TableLoadContext commit_builder_load_context{dir->Str()}; std::map commit_builder_options; CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); ASSERT_OK(commit->Commit(messages)); // The same for the snapshot half of the commit interface. @@ -2981,22 +2981,22 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { // Closing after a prepared commit must not take back what the commit just published. ASSERT_OK(write->Close()); - TableLoadOptions scan_builder_load_options{dir->Str()}; + TableLoadContext scan_builder_load_context{dir->Str()}; std::map scan_builder_options; ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1u); - TableLoadOptions read_builder_load_options{dir->Str()}; + TableLoadContext read_builder_load_context{dir->Str()}; std::map read_builder_options; ReadContextBuilder read_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); @@ -3034,11 +3034,11 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // The same location by path alone is not a table at all: there is no schema under it to say // what kind of table it is. - TableLoadOptions path_builder_load_options{data_dir->Str()}; + TableLoadContext path_builder_load_context{data_dir->Str()}; std::map path_builder_options; ScanContextBuilder path_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr path_context, path_builder.Finish()); - ASSERT_NOK(TableScan::Create(path_builder_load_options, std::move(path_context), + ASSERT_NOK(TableScan::Create(path_builder_load_context, std::move(path_context), path_builder_options)); // Options given at the call still win over the ones the schema stored, so a file is rolled @@ -3218,35 +3218,35 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr MakeSchema(), /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}})); - TableLoadOptions read_builder_load_options{dir->Str()}; + TableLoadContext read_builder_load_context{dir->Str()}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options["branch"] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(read_builder_load_options, std::move(read_context), read_builder_options), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options), "has no branches"); // A scan takes its branch from the option rather than from a `WithBranch()` of its own, which // is the same branch by another name and gets the same answer. - TableLoadOptions scan_builder_load_options{dir->Str()}; + TableLoadContext scan_builder_load_context{dir->Str()}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options = {{Options::BRANCH, "dev"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options), "has no branches"); // The main branch is untouched by any of it. - TableLoadOptions main_builder_load_options{dir->Str()}; + TableLoadContext main_builder_load_context{dir->Str()}; std::map main_builder_options; ScanContextBuilder main_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, - TableScan::Create(main_builder_load_options, std::move(main_context), + TableScan::Create(main_builder_load_context, std::move(main_context), main_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, main_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector rows, ReadAll(table, plan->Splits())); @@ -3264,14 +3264,14 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // `target-file-row-num` is stored nowhere in this table's schema, so a file would hold every // row. Given at the call it has to roll a file per row, as it does for a managed table. - TableLoadOptions write_builder_load_options{dir->Str()}; + TableLoadContext write_builder_load_context{dir->Str()}; std::map write_builder_options; WriteContextBuilder write_builder("test-user"); write_builder_options = {{Options::TARGET_FILE_ROW_NUM, "1"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); for (int32_t i = 0; i < 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, @@ -3284,27 +3284,27 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // Published through the generic commit, or nothing below would see the rows: until the commit // renames them the files sit in the hidden `_temporary` directory a scan skips. - TableLoadOptions generic_commit_builder_load_options{dir->Str()}; + TableLoadContext generic_commit_builder_load_context{dir->Str()}; std::map generic_commit_builder_options; CommitContextBuilder generic_commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit_context, generic_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr generic_commit, - FileStoreCommit::Create(generic_commit_builder_load_options, + FileStoreCommit::Create(generic_commit_builder_load_context, std::move(generic_commit_context), generic_commit_builder_options)); ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); // A branch this table has no schema on is a managed table as far as dispatch is concerned, so // the format branch must not answer for it. - TableLoadOptions branch_builder_load_options{dir->Str()}; + TableLoadContext branch_builder_load_context{dir->Str()}; std::map branch_builder_options; ReadContextBuilder branch_builder; branch_builder_options["branch"] = "nosuchbranch"; ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); - ASSERT_NOK(TableRead::Create(branch_builder_load_options, std::move(branch_context), + ASSERT_NOK(TableRead::Create(branch_builder_load_context, std::move(branch_context), branch_builder_options)); // A caller-held schema is used instead of reading one from under the path. @@ -3313,14 +3313,14 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { schema_manager.Latest()); ASSERT_TRUE(latest.has_value()); ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - TableLoadOptions seeded_builder_load_options{dir->Str()}; + TableLoadContext seeded_builder_load_context{dir->Str()}; std::map seeded_builder_options; ScanContextBuilder seeded_builder; - seeded_builder_load_options.table_schema_json = schema_json; + seeded_builder_load_context.table_schema_json = schema_json; ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_context, seeded_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_scan, - TableScan::Create(seeded_builder_load_options, std::move(seeded_context), + TableScan::Create(seeded_builder_load_context, std::move(seeded_context), seeded_builder_options)); // Planning, not just dispatch: handing the schema over must not turn the `schema` directory // the table keeps under its own path into data, which a plan would then read as a data file. @@ -3332,14 +3332,14 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { // `type` is structural, so an option given at the call must not decide what kind of table // this is. Checked by reading the rows back rather than by counting splits: three small files // are packed into one split, so a split count says nothing about how many there are. - TableLoadOptions retyped_builder_load_options{dir->Str()}; + TableLoadContext retyped_builder_load_context{dir->Str()}; std::map retyped_builder_options; ScanContextBuilder retyped_builder; retyped_builder_options = {{Options::TYPE, "table"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_context, retyped_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_scan, - TableScan::Create(retyped_builder_load_options, std::move(retyped_context), + TableScan::Create(retyped_builder_load_context, std::move(retyped_context), retyped_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr retyped_plan, retyped_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector retyped_rows, @@ -3387,12 +3387,12 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { MakeBatch({1, 2, 3}, {"alice", "bob", "carol"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(WriteAndCommit(table, std::move(batch))); - TableLoadOptions scan_builder_load_options{dir->Str()}; + TableLoadContext scan_builder_load_context{dir->Str()}; std::map scan_builder_options; ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); @@ -3442,7 +3442,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { // A projected read schema can rename a column, prune a nested one and give it metadata of its // own; a format table's projection is a list of top-level names. - TableLoadOptions read_schema_builder_load_options{dir->Str()}; + TableLoadContext read_schema_builder_load_context{dir->Str()}; std::map read_schema_builder_options; ReadContextBuilder read_schema_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> projected_schema, table->GetArrowSchema()); @@ -3450,38 +3450,38 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { ASSERT_OK_AND_ASSIGN(std::unique_ptr read_schema_context, read_schema_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(read_schema_builder_load_options, std::move(read_schema_context), + TableRead::Create(read_schema_builder_load_context, std::move(read_schema_context), read_schema_builder_options), "does not take a projected read schema"); - TableLoadOptions streaming_builder_load_options{dir->Str()}; + TableLoadContext streaming_builder_load_context{dir->Str()}; std::map streaming_builder_options; ScanContextBuilder streaming_builder; streaming_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(streaming_builder_load_options, + ASSERT_NOK_WITH_MSG(TableScan::Create(streaming_builder_load_context, std::move(streaming_context), streaming_builder_options), "nothing for a streaming scan to follow"); - TableLoadOptions predicate_builder_load_options{dir->Str()}; + TableLoadContext predicate_builder_load_context{dir->Str()}; std::map predicate_builder_options; ScanContextBuilder predicate_builder; predicate_builder.SetPredicate(PredicateBuilder::GreaterThan( /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(1))); ASSERT_OK_AND_ASSIGN(std::unique_ptr predicate_context, predicate_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(predicate_builder_load_options, + ASSERT_NOK_WITH_MSG(TableScan::Create(predicate_builder_load_context, std::move(predicate_context), predicate_builder_options), "does not take a predicate"); - TableLoadOptions commit_builder_load_options{dir->Str()}; + TableLoadContext commit_builder_load_context{dir->Str()}; std::map commit_builder_options; CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); // A commit takes batch writes only. The rest of the snapshot half is // `TestTheGenericCommitRefusesEveryCallAboutSnapshots`. @@ -3505,13 +3505,13 @@ TEST(FormatTableTest, TestTheGenericCommitRefusesEveryCallAboutSnapshots) { ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - TableLoadOptions commit_builder_load_options{dir->Str()}; + TableLoadContext commit_builder_load_context{dir->Str()}; std::map commit_builder_options; CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress({}, BATCH_WRITE_COMMIT_IDENTIFIER, @@ -3553,12 +3553,12 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { MakeBatch({9}, {"zoe"}, "20240102", {{"dt", "20240102"}})); ASSERT_OK(WriteAndCommit(table, std::move(other))); - TableLoadOptions write_builder_load_options{dir->Str()}; + TableLoadContext write_builder_load_context{dir->Str()}; std::map write_builder_options; WriteContextBuilder write_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr replacement, MakeBatch({3}, {"carol"}, "20240101", {{"dt", "20240101"}})); @@ -3569,13 +3569,13 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 1u); - TableLoadOptions commit_builder_load_options{dir->Str()}; + TableLoadContext commit_builder_load_context{dir->Str()}; std::map commit_builder_options; CommitContextBuilder commit_builder("test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); // Batch writes only, and refused before anything is cleared: an overwrite that failed on its // arguments must not have deleted the partition it named. diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp index 7ce13b431..0bca6ff9a 100644 --- a/src/paimon/core/table/sink/commit_message_test.cpp +++ b/src/paimon/core/table/sink/commit_message_test.cpp @@ -33,7 +33,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/linked_hash_map.h" @@ -1231,16 +1231,16 @@ TEST(CommitMessageTest, TestSerialize) { /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); - TableLoadOptions context_builder_load_options{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; + TableLoadContext context_builder_load_context{PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user1"); context_builder_options[Options::FILE_FORMAT] = "mock_format"; context_builder_options[Options::TARGET_FILE_SIZE] = "1024"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - std::string root_path = context_builder_load_options.path; + std::string root_path = context_builder_load_context.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); for (size_t i = 0; i < 10240; i++) { diff --git a/src/paimon/core/table/source/append_count_reader_test.cpp b/src/paimon/core/table/source/append_count_reader_test.cpp index 1814f2b39..4a3e00b10 100644 --- a/src/paimon/core/table/source/append_count_reader_test.cpp +++ b/src/paimon/core/table/source/append_count_reader_test.cpp @@ -25,7 +25,7 @@ #include #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" @@ -52,7 +52,7 @@ class AppendCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; if (snapshot_id.has_value()) { @@ -61,7 +61,7 @@ class AppendCountReaderTest : public testing::Test { } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); @@ -138,13 +138,13 @@ TEST_F(AppendCountReaderTest, TestAppendOnlyTableReadCreateCountReaderPredicateN auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5)); diff --git a/src/paimon/core/table/source/filtered_row_id_reader.cpp b/src/paimon/core/table/source/filtered_row_id_reader.cpp index 5742cc4d0..eb44ad114 100644 --- a/src/paimon/core/table/source/filtered_row_id_reader.cpp +++ b/src/paimon/core/table/source/filtered_row_id_reader.cpp @@ -30,7 +30,7 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "fmt/format.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -101,7 +101,7 @@ Result> PinToSnapshot(const std::string& root /// `FilteredRowIdReader` reads with the table that the search loaded, so that a column renamed or /// replaced since then cannot point the filter at another field. Result> CreateReadWithSchema( - const TableLoadOptions& load, std::unique_ptr read_context, + const TableLoadContext& load, std::unique_ptr read_context, const std::shared_ptr& table_schema, const std::map& options, const std::shared_ptr& pool, const std::shared_ptr& executor) { @@ -169,7 +169,7 @@ Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap6 // Unlike the read, the scan may load the latest schema, as it selects files by partition and // row id alone. - TableLoadOptions scan_builder_load{root_path_}; + TableLoadContext scan_builder_load{root_path_}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options = pinned_options; @@ -207,7 +207,7 @@ Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap6 PredicateUtils::CreatePickedFieldFilter( filter_, ObjectUtils::CreateIdentifierToIndexMap(read_fields))); - TableLoadOptions read_builder_load{root_path_}; + TableLoadContext read_builder_load{root_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = pinned_options; diff --git a/src/paimon/core/table/source/pk_count_reader_test.cpp b/src/paimon/core/table/source/pk_count_reader_test.cpp index 832e7125f..580c58f43 100644 --- a/src/paimon/core/table/source/pk_count_reader_test.cpp +++ b/src/paimon/core/table/source/pk_count_reader_test.cpp @@ -25,7 +25,7 @@ #include #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/types/data_field.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/core/schema/schema_manager.h" @@ -52,7 +52,7 @@ class PKCountReaderTest : public testing::Test { protected: Result>> CreateSplits( const std::string& table_path, const std::optional& snapshot_id) { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; if (snapshot_id.has_value()) { @@ -61,7 +61,7 @@ class PKCountReaderTest : public testing::Test { } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); @@ -75,7 +75,7 @@ class PKCountReaderTest : public testing::Test { Result> CreateInternalContext( const std::string& table_path) { - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, @@ -87,11 +87,11 @@ class PKCountReaderTest : public testing::Test { auto internal_context, InternalReadContext::Create( read_context, table_schema, table_schema->Options(), - read_context_builder_load_options.file_system, - read_context_builder_load_options.path, - read_context_builder_load_options.fs_scheme_to_identifier_map, - read_context_builder_load_options.catalog, - read_context_builder_load_options.identifier, read_context_builder_options)); + read_context_builder_load_context.file_system, + read_context_builder_load_context.path, + read_context_builder_load_context.fs_scheme_to_identifier_map, + read_context_builder_load_context.catalog, + read_context_builder_load_context.identifier, read_context_builder_options)); return std::shared_ptr(std::move(internal_context)); } diff --git a/src/paimon/core/table/source/scan_exception_safety_test.cpp b/src/paimon/core/table/source/scan_exception_safety_test.cpp index c453a6933..141863b77 100644 --- a/src/paimon/core/table/source/scan_exception_safety_test.cpp +++ b/src/paimon/core/table/source/scan_exception_safety_test.cpp @@ -30,7 +30,7 @@ #include "arrow/memory_pool.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -118,10 +118,10 @@ Result> OrdinaryScan( std::ifstream schema_file(input.path + "/schema/schema-0"); std::string schema{std::istreambuf_iterator(schema_file), std::istreambuf_iterator()}; - TableLoadOptions builder_load_options{input.path}; + TableLoadContext builder_load_context{input.path}; std::map builder_options; ScanContextBuilder builder; - builder_load_options.table_schema_json = schema; + builder_load_context.table_schema_json = schema; builder_options["scan.snapshot-id"] = input.snapshot; builder.WithMemoryPool(pool).WithExecutor(executor); if (input.cache) { @@ -130,9 +130,9 @@ Result> OrdinaryScan( builder.WithCache(input.cache); } if (input.bucket) builder.SetBucketFilter(*input.bucket); - builder_load_options.file_system = input.file_system; + builder_load_context.file_system = input.file_system; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return TableScan::Create(builder_load_options, std::move(context), builder_options); + return TableScan::Create(builder_load_context, std::move(context), builder_options); } class RejectingPlanningExecutor final : public Executor { diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 68c09eeb6..079e1c72b 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -32,7 +32,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/catalog/catalog_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/core/core_options.h" @@ -66,12 +66,12 @@ TEST(TableReadTest, TestReadWithInvalidContext) { std::string path = paimon::test::GetDataDir() + "/orc/append_10.db/append_10/"; { // read with non-exist field - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "non-exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options), "Get field non-exist failed: not exist in table schema"); } @@ -79,13 +79,13 @@ TEST(TableReadTest, TestReadWithInvalidContext) { // field type and literal type mismatch auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(int64_t{15})); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options), "field f3 has field type BIGINT in literal, mismatch field type DOUBLE in predicate"); } @@ -93,12 +93,12 @@ TEST(TableReadTest, TestReadWithInvalidContext) { // field type in predicate mismatch schema auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BIGINT, Literal(int64_t{15})); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options), "schema type double mismatches predicate field type BIGINT"); } @@ -106,14 +106,14 @@ TEST(TableReadTest, TestReadWithInvalidContext) { // field idx in predicate mismatch in schema auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f3", FieldType::DOUBLE, Literal(15.0)); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options), "field f3 has field idx 0 in input schema, mismatch field idx 2 in predicate"); } @@ -121,23 +121,23 @@ TEST(TableReadTest, TestReadWithInvalidContext) { // literal cannot be null auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(FieldType::DOUBLE)); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options), "literal cannot be null in predicate, field name f3"); } { // schema with duplicate field f3 - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f1", "f3"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options), "validate schema failed: read schema has duplicate field f3"); } @@ -145,12 +145,12 @@ TEST(TableReadTest, TestReadWithInvalidContext) { TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"field_no_exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_options, std::move(read_context), + ASSERT_NOK_WITH_MSG(TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options), "Get field field_no_exist failed: not exist in table schema"); } @@ -169,35 +169,35 @@ TEST(TableReadTest, TestFullTextMatchReadEntryAndScoreValidation) { FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), "", "main")); for (bool table_entry : {false, true}) { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0", "f1"}) .SetPredicate(predicate); - auto context_load_options = builder_load_options; + auto context_load_context = builder_load_context; auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); ASSERT_OK(table_entry ? ([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, table->Copy(context_options)); return configured->NewRead(std::move(context)); })() - : TableRead::Create(context_load_options, std::move(context), + : TableRead::Create(context_load_context, std::move(context), context_options)); builder.EnablePreciseFileIndexEvaluate(true) .SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}) .SetPredicate(predicate); - context_load_options.path = builder_load_options.path; - context_load_options.file_system = builder_load_options.file_system; - context_load_options.fs_scheme_to_identifier_map = - builder_load_options.fs_scheme_to_identifier_map; - context_load_options.table_schema_json = builder_load_options.table_schema_json; - context_load_options.catalog = builder_load_options.catalog; - context_load_options.table_id = builder_load_options.table_id; - context_load_options.identifier.reset(); - if (builder_load_options.identifier) { - context_load_options.identifier.emplace(*builder_load_options.identifier); + context_load_context.path = builder_load_context.path; + context_load_context.file_system = builder_load_context.file_system; + context_load_context.fs_scheme_to_identifier_map = + builder_load_context.fs_scheme_to_identifier_map; + context_load_context.table_schema_json = builder_load_context.table_schema_json; + context_load_context.catalog = builder_load_context.catalog; + context_load_context.table_id = builder_load_context.table_id; + context_load_context.identifier.reset(); + if (builder_load_context.identifier) { + context_load_context.identifier.emplace(*builder_load_context.identifier); } context_options = builder_options; ASSERT_OK_AND_ASSIGN(context, builder.Finish()); @@ -206,7 +206,7 @@ TEST(TableReadTest, TestFullTextMatchReadEntryAndScoreValidation) { table->Copy(context_options)); return configured->NewRead(std::move(context)); })() - : TableRead::Create(context_load_options, + : TableRead::Create(context_load_context, std::move(context), context_options), "Score-producing FullTextMatch cannot appear under OR"); } @@ -224,7 +224,7 @@ TEST(TableReadTest, TestPerFieldScoreReadEntry) { FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), "", "main")); for (bool table_entry : {false, true}) { - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(search).SetReadFieldNames( @@ -234,7 +234,7 @@ TEST(TableReadTest, TestPerFieldScoreReadEntry) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, table->Copy(builder_options)); return configured->NewRead(std::move(context)); })() - : TableRead::Create(builder_load_options, std::move(context), + : TableRead::Create(builder_load_context, std::move(context), builder_options)); } } @@ -242,14 +242,14 @@ TEST(TableReadTest, TestPerFieldScoreReadEntry) { TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) { const std::string path = GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; - TableLoadOptions scan_builder_load_options{path}; + TableLoadContext scan_builder_load_context{path}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); std::shared_ptr selected_split; @@ -279,7 +279,7 @@ TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) auto scalar = PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0)); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({text, scalar})); - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder_options[Options::READ_BATCH_SIZE] = "1"; @@ -289,7 +289,7 @@ TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, - TableRead::Create(builder_load_options, std::move(context), builder_options)); + TableRead::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(selected_split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -308,7 +308,7 @@ TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) TEST(TableReadTest, TestCreateKeyValueTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); @@ -318,7 +318,7 @@ TEST(TableReadTest, TestCreateKeyValueTableRead) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); auto key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read); @@ -329,7 +329,7 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; auto search = std::make_shared("f0", R"({"match":{"query":"Alice"}})", /*limit=*/1); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true) @@ -337,7 +337,7 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { .SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "CreateCountReader does not support precise file-index evaluation"); @@ -348,7 +348,7 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { auto search = std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true) @@ -356,7 +356,7 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { .SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "CreateCountReader does not support precise file-index evaluation"); @@ -365,12 +365,12 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { TEST(TableReadTest, TestPreciseEvaluationRejectsSystemAndFormatTables) { std::string path = GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions system_builder_load_options{path + "$snapshots"}; + TableLoadContext system_builder_load_context{path + "$snapshots"}; std::map system_builder_options; ReadContextBuilder system_builder; system_builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto system_context, system_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(system_builder_load_options, std::move(system_context), + ASSERT_NOK_WITH_MSG(TableRead::Create(system_builder_load_context, std::move(system_context), system_builder_options), "Precise file-index evaluation is not supported for system tables"); @@ -385,13 +385,13 @@ TEST(TableReadTest, TestPreciseEvaluationRejectsSystemAndFormatTables) { Identifier("db", "table"), {})); for (bool loaded_table : {false, true}) { ReadContextBuilder builder; - TableLoadOptions builder_load_options{directory->Str()}; + TableLoadContext builder_load_context{directory->Str()}; std::map builder_options; builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); ASSERT_NOK_WITH_MSG(loaded_table ? format_table->NewRead(std::move(context)) - : TableRead::Create(builder_load_options, std::move(context)), + : TableRead::Create(builder_load_context, std::move(context)), "Precise file-index evaluation is not supported for format tables"); } } @@ -402,7 +402,7 @@ TEST(TableReadTest, TestPreciseScalarEvaluationRejectsUnsupportedTableModes) { GetDataDir() + (primary_key ? "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality" : "/orc/append_09.db/append_09"); - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f3"}).SetPredicate( @@ -420,7 +420,7 @@ TEST(TableReadTest, TestPreciseScalarEvaluationRejectsUnsupportedTableModes) { ASSERT_TRUE(latest); ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, latest.value()->MergeOptions(builder_options)); - ASSERT_OK_AND_ASSIGN(builder_load_options.table_schema_json, schema->GetJsonSchema()); + ASSERT_OK_AND_ASSIGN(builder_load_context.table_schema_json, schema->GetJsonSchema()); builder_options.clear(); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); const std::string expected_error = @@ -429,7 +429,7 @@ TEST(TableReadTest, TestPreciseScalarEvaluationRejectsUnsupportedTableModes) { : "Precise file-index evaluation does not support data-evolution or " "real-time reads"; ASSERT_NOK_WITH_MSG( - TableRead::Create(builder_load_options, std::move(context), builder_options), + TableRead::Create(builder_load_context, std::move(context), builder_options), expected_error); } } @@ -440,14 +440,14 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { auto search = std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f0"}).SetVectorSearch( search); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, - TableRead::Create(context_builder_load_options, std::move(context), + TableRead::Create(context_builder_load_context, std::move(context), context_builder_options)); BinaryRow partition = BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get()); @@ -468,7 +468,7 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { TEST(TableReadTest, TestCreateAppendOnlyTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); @@ -478,7 +478,7 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); @@ -486,7 +486,7 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); @@ -498,7 +498,7 @@ TEST(TableReadTest, TestMergeOptions) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); auto append_only_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(append_only_table_read); @@ -528,10 +528,10 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { ASSERT_OK_AND_ASSIGN(std::string schema_json, table_schema->ToJsonString()); for (bool explicit_header : {false, true}) { - TableLoadOptions builder_load_options{"warehouse/db.db/table"}; + TableLoadContext builder_load_context{"warehouse/db.db/table"}; std::map builder_options; ReadContextBuilder builder; - builder_load_options.table_schema_json = schema_json; + builder_load_context.table_schema_json = schema_json; if (explicit_header) { builder_options[read_via_option] = "read-view"; @@ -540,14 +540,14 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { // Exercise the actual table/read merge before creating the dependency context. ASSERT_OK_AND_ASSIGN( std::unique_ptr table_read, - TableRead::Create(builder_load_options, std::move(read_context), builder_options)); + TableRead::Create(builder_load_context, std::move(read_context), builder_options)); auto* append_only_table_read = dynamic_cast(table_read.get()); ASSERT_NE(append_only_table_read, nullptr); const auto& internal_context = append_only_table_read->context_; auto dependency_context = internal_context->CreateDependencyCatalogContext(); auto expected_options = table_options; - expected_options["path"] = builder_load_options.path; + expected_options["path"] = builder_load_context.path; expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 4e4b8f062..d77d4040b 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -25,7 +25,7 @@ #include #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/metrics.h" @@ -61,13 +61,13 @@ TEST(TableScanTest, TestDefaultMetricsSnapshot) { TEST(TableScanTest, TestNoSnapshot) { std::string path = paimon::test::GetDataDir() + "/orc/append_table_with_nested_type.db/append_table_with_nested_type/"; - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::FILE_FORMAT] = "orc"; ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_FALSE(plan->SnapshotId()); @@ -76,14 +76,14 @@ TEST(TableScanTest, TestNoSnapshot) { TEST(TableScanTest, TestNonExistTable) { std::string path = paimon::test::GetDataDir() + "/non-exist.db/non-exist/"; - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::FILE_FORMAT] = "orc"; ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); ASSERT_NOK_WITH_MSG( - TableScan::Create(builder_load_options, std::move(context), builder_options), + TableScan::Create(builder_load_context, std::move(context), builder_options), "not found latest schema"); } @@ -91,7 +91,7 @@ TEST(TableScanTest, TestStreamingSnapshotsRemoved) { auto dir = UniqueTestDirectory::Create("local"); ASSERT_TRUE(dir); ASSERT_TRUE(TestUtil::CopyDirectory(GetDataDir() + "/orc/append_09.db/append_09", dir->Str())); - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::FILE_FORMAT] = "orc"; @@ -100,7 +100,7 @@ TEST(TableScanTest, TestStreamingSnapshotsRemoved) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_scan, - TableScan::Create(builder_load_options, std::move(context), builder_options)); + TableScan::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(std::shared_ptr waiting_plan, table_scan->CreatePlan()); @@ -116,7 +116,7 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { std::string path = GetDataDir() + "/orc/append_09.db/append_09"; for (const std::string next_snapshot_id : {"6", "7"}) { SCOPED_TRACE(next_snapshot_id); - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::FILE_FORMAT] = "orc"; @@ -128,7 +128,7 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_scan, - TableScan::Create(builder_load_options, std::move(context), builder_options)); + TableScan::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr initial_plan, table_scan->CreatePlan()); ASSERT_TRUE(initial_plan->Splits().empty()); @@ -149,13 +149,13 @@ TEST(TableScanTest, TestStreamingNextSnapshotBoundary) { TEST(TableScanTest, TestPkSchemaEvolutionScan) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table/"; - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::FILE_FORMAT] = "orc"; ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_TRUE(plan->SnapshotId()); @@ -186,7 +186,7 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::FILE_FORMAT] = "orc"; @@ -195,7 +195,7 @@ TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_NOK_WITH_MSG( - TableScan::Create(builder_load_options, std::move(context), builder_options), + TableScan::Create(builder_load_context, std::move(context), builder_options), "read-optimized system table does not support streaming scan for primary " "key table"); } diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 1e1565b08..ac8e74d66 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -31,7 +31,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/generic_row.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" diff --git a/src/paimon/core/table/table_factory.cpp b/src/paimon/core/table/table_factory.cpp index 37b9bb649..417dd0480 100644 --- a/src/paimon/core/table/table_factory.cpp +++ b/src/paimon/core/table/table_factory.cpp @@ -52,7 +52,7 @@ namespace paimon { namespace { -Status CheckCatalogTable(const TableLoadOptions& load, const std::shared_ptr
& table) { +Status CheckCatalogTable(const TableLoadContext& load, const std::shared_ptr
& table) { if (load.table_id && load.table_id != table->CatalogUuid()) { return Status::Invalid("the expected table id does not match the catalog table id"); } @@ -75,7 +75,7 @@ Status CheckCatalogTable(const TableLoadOptions& load, const std::shared_ptr> TableFactory::Create( - const TableLoadOptions& load, const std::map& overrides, + const TableLoadContext& load, const std::map& overrides, const char* operation) { if (load.catalog == nullptr && load.identifier) { return Status::Invalid("a table identifier requires a non-null catalog"); @@ -263,75 +263,75 @@ Result> TableFactory::Create( } Result> TableRead::Create( - const TableLoadOptions& load_options, std::unique_ptr context, + const TableLoadContext& load_context, std::unique_ptr context, const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); - if (load_options.table_id && !load_options.catalog) { + if (load_context.table_id && !load_context.catalog) { return Status::Invalid("a table id requires a catalog or an external REST commit request"); } if (context->GetRealtimeContext()) { PAIMON_ASSIGN_OR_RAISE(std::string branch, - BranchManager::ResolveBranch(load_options.identifier, + BranchManager::ResolveBranch(load_context.identifier, dynamic_options, std::nullopt, "read")); PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, - TableFactory::Create(load_options, dynamic_options, "read")); + TableFactory::Create(load_context, dynamic_options, "read")); PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); return table->NewRead(std::move(context)); } Result> TableScan::Create( - const TableLoadOptions& load_options, std::unique_ptr context, + const TableLoadContext& load_context, std::unique_ptr context, const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - if (load_options.table_id && !load_options.catalog) { + if (load_context.table_id && !load_context.catalog) { return Status::Invalid("a table id requires a catalog or an external REST commit request"); } if (context->GetRealtimeContext()) { PAIMON_ASSIGN_OR_RAISE(std::string branch, - BranchManager::ResolveBranch(load_options.identifier, + BranchManager::ResolveBranch(load_context.identifier, dynamic_options, std::nullopt, "scan")); PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, - TableFactory::Create(load_options, dynamic_options, "scan")); + TableFactory::Create(load_context, dynamic_options, "scan")); PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); return table->NewScan(std::move(context)); } Result> FileStoreWrite::Create( - const TableLoadOptions& load_options, std::unique_ptr context, + const TableLoadContext& load_context, std::unique_ptr context, const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); - if (load_options.table_id && !load_options.catalog) { + if (load_context.table_id && !load_context.catalog) { return Status::Invalid("a table id requires a catalog or an external REST commit request"); } if (context->GetRealtimeContext()) { PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - load_options.identifier, dynamic_options, + load_context.identifier, dynamic_options, std::nullopt, "write")); PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch)); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, - TableFactory::Create(load_options, dynamic_options, "write")); + TableFactory::Create(load_context, dynamic_options, "write")); PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); return table->NewWrite(std::move(context)); } Result> FileStoreCommit::Create( - const TableLoadOptions& load_options, std::unique_ptr context, + const TableLoadContext& load_context, std::unique_ptr context, const std::map& dynamic_options) { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); - if (load_options.catalog && context->UseRESTCatalogCommit()) { + if (load_context.catalog && context->UseRESTCatalogCommit()) { return Status::Invalid( "a commit cannot both use a catalog and build an external REST request"); } - if (load_options.table_id && !load_options.catalog && !context->UseRESTCatalogCommit()) { + if (load_context.table_id && !load_context.catalog && !context->UseRESTCatalogCommit()) { return Status::Invalid("a table id requires a catalog or an external REST commit request"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, - TableFactory::Create(load_options, dynamic_options, "commit")); + TableFactory::Create(load_context, dynamic_options, "commit")); PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); return table->NewCommit(std::move(context)); } diff --git a/src/paimon/core/table/table_factory.h b/src/paimon/core/table/table_factory.h index 564a6baca..bd5f14685 100644 --- a/src/paimon/core/table/table_factory.h +++ b/src/paimon/core/table/table_factory.h @@ -22,7 +22,7 @@ #include #include -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/result.h" namespace paimon { @@ -35,7 +35,7 @@ class TableFactory { ~TableFactory() = delete; static Result> Create( - const TableLoadOptions& load_options, + const TableLoadContext& load_context, const std::map& dynamic_options, const char* operation); }; } // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index cf7553310..2a569b5f3 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -34,7 +34,7 @@ #include "arrow/ipc/json_simple.h" #include "fmt/format.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/catalog/catalog_context.h" #include "paimon/common/fs/resolving_file_system.h" @@ -218,7 +218,7 @@ TEST(TableTest, TestCopyRejectsBucketKeyChangesWithoutLosingRows) { ASSERT_FALSE(plan->Splits().empty()); // A static factory must validate against the persisted schema, before merging overrides. - TableLoadOptions load{table->GetRootPath()}; + TableLoadContext load{table->GetRootPath()}; const std::map invalid = {{Options::BUCKET_KEY, "b"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(load, std::move(scan_context), invalid), "bucket-key"); @@ -275,7 +275,7 @@ TEST(TableTest, TestSystemTableCopiesPreserveManagedOptionRules) { {}, {"a", "b"}, {{Options::BUCKET, "2"}, {Options::BUCKET_KEY, "a"}})); for (const std::string kind : {"ro", "audit_log", "binlog"}) { SCOPED_TRACE(kind); - TableLoadOptions load{dir->Str() + "$" + kind}; + TableLoadContext load{dir->Str() + "$" + kind}; ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(load, {}, "read")); ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "b"}}), "bucket-key"); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, @@ -414,9 +414,9 @@ TEST(TableTest, TestScanWithMainBranchAlias) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(identifier)); for (bool explicit_main : {false, true}) { SCOPED_TRACE(explicit_main); - TableLoadOptions load_options{table_path}; - load_options.catalog = catalog; - load_options.identifier.emplace(identifier); + TableLoadContext load_context{table_path}; + load_context.catalog = catalog; + load_context.identifier.emplace(identifier); std::map dynamic_options; if (explicit_main) { dynamic_options[Options::BRANCH] = "main"; @@ -425,7 +425,7 @@ TEST(TableTest, TestScanWithMainBranchAlias) { ASSERT_OK_AND_ASSIGN(std::unique_ptr static_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr static_scan, - TableScan::Create(load_options, std::move(static_context), dynamic_options)); + TableScan::Create(load_context, std::move(static_context), dynamic_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr static_plan, static_scan->CreatePlan()); ASSERT_TRUE(static_plan->Splits().empty()); @@ -511,7 +511,7 @@ TEST(TableTest, TestGetTableDispatchesByTableType) { ASSERT_EQ((std::vector{"key", "value"}), system_table->GetSchema()->FieldNames()); ASSERT_EQ(nullptr, std::dynamic_pointer_cast(system_table->GetSchema())); for (bool direct : {false, true}) { - TableLoadOptions context_builder_load_options{managed_dir + "$options"}; + TableLoadContext context_builder_load_context{managed_dir + "$options"}; std::map context_builder_options; ReadContextBuilder context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, @@ -521,7 +521,7 @@ TEST(TableTest, TestGetTableDispatchesByTableType) { system_table->Copy(context_builder_options)); return configured->NewRead(std::move(context)); })() - : TableRead::Create(context_builder_load_options, std::move(context), + : TableRead::Create(context_builder_load_context, std::move(context), context_builder_options); ASSERT_NOK_WITH_MSG(read, "Precise file-index evaluation is not supported for system tables"); @@ -747,7 +747,7 @@ std::string CompoundFilter(const std::string& function, const std::string& left, } Result> ReadAll( - const TableLoadOptions& load, std::unique_ptr context, + const TableLoadContext& load, std::unique_ptr context, const std::vector>& splits, const std::map& options = {}) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, @@ -776,7 +776,7 @@ Result> PlanQueryAuth(const std::string& path, const std::shared_ptr& catalog, const std::shared_ptr& predicate, std::optional limit) { - TableLoadOptions builder_load{path}; + TableLoadContext builder_load{path}; std::map builder_options; ScanContextBuilder builder; builder_load.catalog = catalog; @@ -796,12 +796,12 @@ Result> PlanQueryAuth(const std::string& path, Result> PlanAndReadAll( const std::string& path, const std::shared_ptr& catalog, const std::vector& read_fields = {}) { - TableLoadOptions scan_builder_load{path}; + TableLoadContext scan_builder_load{path}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_load.catalog = catalog; scan_builder_load.identifier.emplace(Identifier("foo", "bar")); - TableLoadOptions read_builder_load{path}; + TableLoadContext read_builder_load{path}; std::map read_builder_options; ReadContextBuilder read_builder; if (!read_fields.empty()) { @@ -865,7 +865,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { R"({"kind": "LEAF", "transform": {"name": "FIELD_REF", "fieldRef": {"index": 0,)" R"( "name": "id", "type": "INT NOT NULL"}}, "function": "GREATER_THAN", "literals": [1]})"}; - TableLoadOptions scan_context_load_6{path}; + TableLoadContext scan_context_load_6{path}; scan_context_load_6.catalog = catalog; scan_context_load_6.identifier.emplace(identifier); @@ -883,7 +883,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { ASSERT_NE(nullptr, std::dynamic_pointer_cast(split)); } - TableLoadOptions read_context_load_7{path}; + TableLoadContext read_context_load_7{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -893,14 +893,14 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { AssertColumn(*rows, "phone", R"(["139", null])"); AssertColumn(*rows, "salary", "[null, null]"); - TableLoadOptions read_context_load_8{path}; + TableLoadContext read_context_load_8{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"name"}).Finish()); ASSERT_OK_AND_ASSIGN(rows, ReadAll(read_context_load_8, std::move(read_context), plan->Splits(), {})); AssertColumn(*rows, "name", R"(["bob", "carol"])"); ASSERT_EQ(nullptr, rows->GetFieldByName("id")); - TableLoadOptions read_context_load_9{path}; + TableLoadContext read_context_load_9{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldIds({1}).Finish()); ASSERT_OK_AND_ASSIGN(rows, @@ -908,7 +908,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { AssertColumn(*rows, "name", R"(["bob", "carol"])"); ASSERT_EQ(nullptr, rows->GetFieldByName("id")); - TableLoadOptions read_context_load_10{path}; + TableLoadContext read_context_load_10{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder() .SetReadFieldNames({"name", "phone"}) @@ -938,7 +938,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { ASSERT_EQ(original->GetScope(), auth_split->GetScope()); deserialized.push_back(copy); } - TableLoadOptions read_context_load_11{path}; + TableLoadContext read_context_load_11{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(rows, @@ -954,7 +954,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { ASSERT_OK_AND_ASSIGN( std::shared_ptr java_planned_split, Split::Deserialize(java_planned.data(), java_planned.size(), GetDefaultPool())); - TableLoadOptions read_context_load_12{path}; + TableLoadContext read_context_load_12{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"name"}).Finish()); ASSERT_NOK_WITH_MSG( @@ -966,7 +966,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "Query auth rules changed"); ASSERT_EQ(2u, catalog->auth_identifiers.size()); - TableLoadOptions scan_context_load_13{path}; + TableLoadContext scan_context_load_13{path}; scan_context_load_13.catalog = catalog; scan_context_load_13.identifier.emplace(identifier); @@ -976,7 +976,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { TableScan::Create(scan_context_load_13, std::move(scan_context), {})); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "masked partition key 'phone'"); - TableLoadOptions read_context_load_14{path}; + TableLoadContext read_context_load_14{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, @@ -1007,7 +1007,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { catalog->filter = { R"({"kind": "LEAF", "transform": {"name": "FIELD_REF", "fieldRef": {"index": 9,)" R"( "name": "gone", "type": "INT"}}, "function": "IS_NULL", "literals": []})"}; - TableLoadOptions scan_context_load_15{path}; + TableLoadContext scan_context_load_15{path}; scan_context_load_15.catalog = catalog; scan_context_load_15.identifier.emplace(identifier); @@ -1019,7 +1019,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { "schema 0): Row filter references column 'gone' which does not exist"); catalog->filter = {}; catalog->column_masking = {{"name", R"({"name": "UPPER", "inputs": []})"}}; - TableLoadOptions scan_context_load_16{path}; + TableLoadContext scan_context_load_16{path}; scan_context_load_16.catalog = catalog; scan_context_load_16.identifier.emplace(identifier); @@ -1033,7 +1033,7 @@ TEST(TableTest, TestQueryAuthTableAppliesTheRulesOfItsCatalog) { R"({{"kind": "LEAF", "transform": {{"name": "FIELD_REF", "fieldRef": {{"index": 0,)" R"( "name": "id", "type": "INT NOT NULL"}}}}, "function": "NOT_IN"{}}})", literals)}; - TableLoadOptions scan_context_load_17{path}; + TableLoadContext scan_context_load_17{path}; scan_context_load_17.catalog = catalog; scan_context_load_17.identifier.emplace(identifier); @@ -1058,7 +1058,7 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); catalog->column_masking = {{"phone", R"({"name": "NULL"})"}}; - TableLoadOptions scan_context_load_18{path}; + TableLoadContext scan_context_load_18{path}; scan_context_load_18.catalog = catalog; scan_context_load_18.identifier.emplace(identifier); @@ -1071,7 +1071,7 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { ->GetAuthResult() ->HasRules()); catalog->column_masking = {}; - TableLoadOptions scan_context_load_19{path}; + TableLoadContext scan_context_load_19{path}; scan_context_load_19.catalog = catalog; scan_context_load_19.identifier.emplace(identifier); @@ -1080,7 +1080,7 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { TableScan::Create(scan_context_load_19, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr all_plan, scan->CreatePlan()); catalog->column_masking = {{"phone", R"({"name": "NULL"})"}}; - TableLoadOptions scan_context_load_20{path}; + TableLoadContext scan_context_load_20{path}; scan_context_load_20.catalog = catalog; scan_context_load_20.identifier.emplace(identifier); @@ -1093,7 +1093,7 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { "OR", ConstantFilter("TRUE"), R"({"kind": "LEAF", "transform": {"name": "FIELD_REF", "fieldRef": {"index": 2,)" R"( "name": "phone", "type": "STRING"}}, "function": "IS_NULL", "literals": []})")}; - TableLoadOptions scan_context_load_21{path}; + TableLoadContext scan_context_load_21{path}; scan_context_load_21.catalog = catalog; scan_context_load_21.identifier.emplace(identifier); @@ -1103,14 +1103,14 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { ASSERT_OK_AND_ASSIGN(std::shared_ptr folded_plan, scan->CreatePlan()); catalog->filter = {}; catalog->column_masking = {{"phone", R"({"name": "NULL"})"}}; - TableLoadOptions scan_context_load_22{path}; + TableLoadContext scan_context_load_22{path}; scan_context_load_22.catalog = catalog; scan_context_load_22.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().SetReadFieldNames({"name"}).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr unplanned_scan, TableScan::Create(scan_context_load_22, std::move(scan_context), {})); - TableLoadOptions scan_context_load_23{path}; + TableLoadContext scan_context_load_23{path}; scan_context_load_23.catalog = catalog; scan_context_load_23.identifier.emplace(identifier); @@ -1151,7 +1151,7 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { "Query auth rules reference column 'secret', which the schema the scan " "planned with did not hold"); - TableLoadOptions read_context_load_24{path}; + TableLoadContext read_context_load_24{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().SetReadFieldNames({"name"}).Finish()); @@ -1159,14 +1159,14 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { ReadAll(read_context_load_24, std::move(read_context), name_plan->Splits(), {}), "Column 'name' is read as field 2, but the scan authorized field 1 under " "that name"); - TableLoadOptions read_context_load_25{path}; + TableLoadContext read_context_load_25{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"secret"}).Finish()); ASSERT_NOK_WITH_MSG( ReadAll(read_context_load_25, std::move(read_context), all_plan->Splits(), {}), "Column 'secret' is read, but the scan did not ask the catalog to " "authorize it"); - TableLoadOptions read_context_load_26{path}; + TableLoadContext read_context_load_26{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"id", "salary"}).Finish()); @@ -1174,14 +1174,14 @@ TEST(TableTest, TestQueryAuthTableRefusesColumnsChangedSinceThePlan) { std::shared_ptr rows, ReadAll(read_context_load_26, std::move(read_context), all_plan->Splits(), {})); AssertColumn(*rows, "id", "[1, 2, 3]"); - TableLoadOptions read_context_load_27{path}; + TableLoadContext read_context_load_27{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"id"}).Finish()); ASSERT_NOK_WITH_MSG( ReadAll(read_context_load_27, std::move(read_context), masked_plan->Splits(), {}), "Query auth rules reference column 'phone' which the schema being read " "exposes as 'name' (renamed since)"); - TableLoadOptions read_context_load_28{path}; + TableLoadContext read_context_load_28{path}; ASSERT_OK_AND_ASSIGN(read_context, ReadContextBuilder().SetReadFieldNames({"id"}).Finish()); ASSERT_NOK_WITH_MSG( @@ -1212,7 +1212,7 @@ TEST(TableTest, TestQueryAuthTableAsksForTheRulesOfTheBranchPlanned) { CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); catalog->column_masking = {{"phone", R"({"name": "NULL"})"}}; - TableLoadOptions scan_context_load_29{path}; + TableLoadContext scan_context_load_29{path}; std::map scan_context_options_29; scan_context_load_29.catalog = catalog; scan_context_load_29.identifier.emplace(Identifier("foo", "bar")); @@ -1231,7 +1231,7 @@ TEST(TableTest, TestQueryAuthTableAsksForTheRulesOfTheBranchPlanned) { ASSERT_EQ(std::vector({Identifier("foo", "bar$branch_dev")}), catalog->location_identifiers); - TableLoadOptions read_context_load_30{path}; + TableLoadContext read_context_load_30{path}; std::map read_context_options_30; read_context_options_30[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); @@ -1260,7 +1260,7 @@ TEST(TableTest, TestQueryAuthTableIsNotSkippedThroughAStaleCachedSchema) { document.Accept(writer); const std::string stale_schema = buffer.GetString(); - TableLoadOptions scan_context_load_31{path}; + TableLoadContext scan_context_load_31{path}; scan_context_load_31.table_schema_json = stale_schema; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); @@ -1269,7 +1269,7 @@ TEST(TableTest, TestQueryAuthTableIsNotSkippedThroughAStaleCachedSchema) { ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); - TableLoadOptions scan_context_load_32{path}; + TableLoadContext scan_context_load_32{path}; scan_context_load_32.catalog = catalog; scan_context_load_32.identifier.emplace(Identifier("foo", "bar")); @@ -1277,7 +1277,7 @@ TEST(TableTest, TestQueryAuthTableIsNotSkippedThroughAStaleCachedSchema) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(scan_context_load_32, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - TableLoadOptions read_context_load_33{path}; + TableLoadContext read_context_load_33{path}; read_context_load_33.table_schema_json = stale_schema; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); @@ -1292,7 +1292,7 @@ TEST(TableTest, TestQueryAuthTableIsNotSkippedThroughAStaleCachedSchema) { .CreateTable(QueryAuthTableSchema(), /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::QUERY_AUTH_ENABLED, "false"}}) .status()); - TableLoadOptions read_context_load_34{path}; + TableLoadContext read_context_load_34{path}; std::map read_context_options_34; read_context_options_34[Options::BRANCH] = "dev"; read_context_load_34.table_schema_json = stale_schema; @@ -1319,7 +1319,7 @@ TEST(TableTest, TestQueryAuthScanDoesNotPruneByTheRawValuesOfAMaskedColumn) { auto read_with = [&](const std::shared_ptr& predicate, const std::shared_ptr& plan, bool enable_predicate_filter = true) -> Result> { - TableLoadOptions context_load_35{path}; + TableLoadContext context_load_35{path}; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, ReadContextBuilder() @@ -1331,7 +1331,7 @@ TEST(TableTest, TestQueryAuthScanDoesNotPruneByTheRawValuesOfAMaskedColumn) { const std::shared_ptr phone_is_139 = PredicateBuilder::Equal( /*field_index=*/2, "phone", FieldType::STRING, Literal(FieldType::STRING, "139", 3)); - TableLoadOptions scan_context_load_36{path}; + TableLoadContext scan_context_load_36{path}; scan_context_load_36.catalog = catalog; scan_context_load_36.identifier.emplace(Identifier("foo", "bar")); @@ -1390,7 +1390,7 @@ TEST(TableTest, TestQueryAuthReadRefusesAColumnTheScanDidNotAuthorize) { auto read_names = [&](bool enable_predicate_filter, const std::vector>& splits) -> Result> { - TableLoadOptions context_load_37{path}; + TableLoadContext context_load_37{path}; PAIMON_ASSIGN_OR_RAISE( std::unique_ptr context, @@ -1405,7 +1405,7 @@ TEST(TableTest, TestQueryAuthReadRefusesAColumnTheScanDidNotAuthorize) { const std::string unauthorized = "Column 'phone' is read, but the scan did not ask the catalog to authorize it"; - TableLoadOptions name_scan_context_load_38{path}; + TableLoadContext name_scan_context_load_38{path}; name_scan_context_load_38.catalog = catalog; name_scan_context_load_38.identifier.emplace(Identifier("foo", "bar")); @@ -1417,7 +1417,7 @@ TEST(TableTest, TestQueryAuthReadRefusesAColumnTheScanDidNotAuthorize) { ASSERT_OK_AND_ASSIGN(std::shared_ptr name_plan, name_scan->CreatePlan()); ASSERT_EQ(std::optional>(std::vector{"name"}), catalog->auth_selects.back()); - TableLoadOptions all_columns_context_load_39{path}; + TableLoadContext all_columns_context_load_39{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr all_columns_context, ReadContextBuilder().Finish()); @@ -1425,7 +1425,7 @@ TEST(TableTest, TestQueryAuthReadRefusesAColumnTheScanDidNotAuthorize) { ReadAll(all_columns_context_load_39, std::move(all_columns_context), name_plan->Splits(), {}), "Column 'id' is read, but the scan did not ask the catalog to authorize it"); - TableLoadOptions name_context_load_40{path}; + TableLoadContext name_context_load_40{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr name_context, ReadContextBuilder().SetReadFieldNames({"name"}).Finish()); @@ -1545,7 +1545,7 @@ TEST(TableTest, TestQueryAuthTableReadsNoRowsUnderAJavaAlwaysFalseFilter) { [&](const std::vector& read_fields) -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, PlanQueryAuth(path, catalog, /*predicate=*/nullptr, std::nullopt)); - TableLoadOptions builder_load{path}; + TableLoadContext builder_load{path}; std::map builder_options; ReadContextBuilder builder; builder.SetReadFieldNames(read_fields); @@ -1643,7 +1643,7 @@ TEST(TableTest, TestQueryAuthDoesNotBringBackRowsADeletionVectorDeletes) { dir->GetFileSystem(), path, "orc", data_messages[0], {{data_message->GetNewFilesIncrement().NewFiles()[0]->file_name, {1}}}, GetDefaultPool())); - TableLoadOptions commit_context_load_42{path}; + TableLoadContext commit_context_load_42{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, CommitContextBuilder("commit_user").Finish()); @@ -1673,7 +1673,7 @@ TEST(TableTest, TestQueryAuthAppliesTheRulesToTheColumnsADataEvolutionTableRebui {R"([[1, "alice", "13800001111", 100], [2, "bob", "13900002222", 200]])"})); const std::string path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - TableLoadOptions write_context_load_43{path}; + TableLoadContext write_context_load_43{path}; ASSERT_OK_AND_ASSIGN( std::unique_ptr write_context, @@ -1700,7 +1700,7 @@ TEST(TableTest, TestQueryAuthAppliesTheRulesToTheColumnsADataEvolutionTableRebui file->AssignFirstRowId(0); } } - TableLoadOptions commit_context_load_44{path}; + TableLoadContext commit_context_load_44{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, CommitContextBuilder("commit_user").Finish()); @@ -1739,7 +1739,7 @@ TEST(TableTest, TestQueryAuthFiltersDictionaryEncodedParquetColumns) { bool enable_predicate_filter) -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, PlanQueryAuth(path, catalog, predicate, std::nullopt)); - TableLoadOptions builder_load{path}; + TableLoadContext builder_load{path}; std::map builder_options; ReadContextBuilder builder; if (predicate != nullptr) { @@ -1829,7 +1829,7 @@ TEST(TableTest, TestQueryAuthScanReadsOnlyTheLocationOfTheAuthorizedTable) { CreateQueryAuthCatalog(*authorized, authorized_dir->GetFileSystem())); const Identifier identifier("foo", "bar"); - TableLoadOptions scan_context_load_46{other->TablePath()}; + TableLoadContext scan_context_load_46{other->TablePath()}; scan_context_load_46.catalog = catalog; scan_context_load_46.identifier.emplace(identifier); @@ -1844,7 +1844,7 @@ TEST(TableTest, TestQueryAuthScanReadsOnlyTheLocationOfTheAuthorizedTable) { {authorized->TablePath() + "/", PathUtil::JoinPath(authorized_dir->Str() + "/", "foo.db//bar")}) { SCOPED_TRACE(path); - TableLoadOptions scan_context_load_47{path}; + TableLoadContext scan_context_load_47{path}; scan_context_load_47.catalog = catalog; scan_context_load_47.identifier.emplace(identifier); @@ -1855,7 +1855,7 @@ TEST(TableTest, TestQueryAuthScanReadsOnlyTheLocationOfTheAuthorizedTable) { } // The identifier is rebound to another table between the scan and its first plan. - TableLoadOptions scan_context_load_48{authorized->TablePath()}; + TableLoadContext scan_context_load_48{authorized->TablePath()}; scan_context_load_48.catalog = catalog; scan_context_load_48.identifier.emplace(identifier); @@ -1871,7 +1871,7 @@ TEST(TableTest, TestQueryAuthScanReadsOnlyTheLocationOfTheAuthorizedTable) { ASSERT_EQ(auth_requests + 1, catalog->auth_identifiers.size()); catalog->table_location.clear(); - TableLoadOptions scan_context_load_49{authorized->TablePath()}; + TableLoadContext scan_context_load_49{authorized->TablePath()}; scan_context_load_49.catalog = catalog; scan_context_load_49.identifier.emplace(identifier); @@ -1911,7 +1911,7 @@ TEST(TableTest, TestQueryAuthReadRejectsSplitsAuthorizedForAnotherTable) { splits.push_back(std::move(restored)); } for (bool with_catalog : {false, true}) { - TableLoadOptions builder_load{other->TablePath()}; + TableLoadContext builder_load{other->TablePath()}; std::map builder_options; ReadContextBuilder builder; if (with_catalog) { @@ -1929,7 +1929,7 @@ TEST(TableTest, TestQueryAuthReadRejectsSplitsAuthorizedForAnotherTable) { // A worker can use an equivalent spelling of the authorized table's location. for (const std::string& path : {authorized->TablePath() + "/", "file://" + authorized->TablePath()}) { - TableLoadOptions context_load_51{path}; + TableLoadContext context_load_51{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -1952,7 +1952,7 @@ TEST(TableTest, TestQueryAuthReadRejectsSplitsAuthorizedForAnotherBranch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanQueryAuth(path, catalog, /*predicate=*/nullptr, std::nullopt)); ASSERT_FALSE(plan->Splits().empty()); - TableLoadOptions context_load_52{path}; + TableLoadContext context_load_52{path}; std::map context_options_52; context_options_52[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); @@ -2013,7 +2013,7 @@ TEST(TableTest, TestCatalogDisablesQueryAuthOnlyForTheTableAtItsLocation) { {std::shared_ptr(auth_catalog), std::shared_ptr(plain_catalog)}) { for (const std::string& suffix : {std::string(), std::string("$files")}) { SCOPED_TRACE(suffix); - TableLoadOptions scan_context_load_53{other->TablePath() + suffix}; + TableLoadContext scan_context_load_53{other->TablePath() + suffix}; scan_context_load_53.catalog = catalog; scan_context_load_53.identifier.emplace(identifier); @@ -2022,7 +2022,7 @@ TEST(TableTest, TestCatalogDisablesQueryAuthOnlyForTheTableAtItsLocation) { ASSERT_NOK_WITH_MSG( TableScan::Create(scan_context_load_53, std::move(scan_context), {}), mismatch("scan")); - TableLoadOptions read_context_load_54{other->TablePath() + suffix}; + TableLoadContext read_context_load_54{other->TablePath() + suffix}; read_context_load_54.catalog = catalog; read_context_load_54.identifier.emplace(identifier); @@ -2032,14 +2032,14 @@ TEST(TableTest, TestCatalogDisablesQueryAuthOnlyForTheTableAtItsLocation) { TableRead::Create(read_context_load_54, std::move(read_context), {}), mismatch("read")); - TableLoadOptions scan_context_load_55{disabled->TablePath() + suffix}; + TableLoadContext scan_context_load_55{disabled->TablePath() + suffix}; scan_context_load_55.catalog = catalog; scan_context_load_55.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); ASSERT_OK( TableScan::Create(scan_context_load_55, std::move(scan_context), {}).status()); - TableLoadOptions read_context_load_56{disabled->TablePath() + suffix}; + TableLoadContext read_context_load_56{disabled->TablePath() + suffix}; read_context_load_56.catalog = catalog; read_context_load_56.identifier.emplace(identifier); @@ -2063,7 +2063,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { const std::string disabled = "Invalid: Option 'query-auth.enabled' is not allowed to be disabled dynamically"; - TableLoadOptions scan_context_load_57{path}; + TableLoadContext scan_context_load_57{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_57, std::move(scan_context), {}), @@ -2072,7 +2072,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { auto plain_catalog = std::make_shared(); plain_catalog->SetFileSystem(dir->GetFileSystem()); plain_catalog->SetTableSchema(checked_pointer_cast(table_schema.value())); - TableLoadOptions scan_context_load_58{path}; + TableLoadContext scan_context_load_58{path}; scan_context_load_58.catalog = plain_catalog; scan_context_load_58.identifier.emplace(identifier); @@ -2080,7 +2080,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_58, std::move(scan_context), {}), "its catalog does not support the query auth interface"); - TableLoadOptions disabling_scan_context_load_59{path}; + TableLoadContext disabling_scan_context_load_59{path}; std::map disabling_scan_context_options_59; disabling_scan_context_options_59[Options::QUERY_AUTH_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::unique_ptr disabling_scan_context, @@ -2089,7 +2089,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { TableScan::Create(disabling_scan_context_load_59, std::move(disabling_scan_context), disabling_scan_context_options_59), disabled); - TableLoadOptions disabling_read_context_load_60{path}; + TableLoadContext disabling_read_context_load_60{path}; std::map disabling_read_context_options_60; disabling_read_context_options_60[Options::QUERY_AUTH_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::unique_ptr disabling_read_context, @@ -2103,7 +2103,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { ASSERT_TRUE(arrow::ExportSchema(*arrow::schema({arrow::field("name", arrow::utf8())}), explicit_read_schema.get()) .ok()); - TableLoadOptions explicit_schema_read_context_load_61{path}; + TableLoadContext explicit_schema_read_context_load_61{path}; ASSERT_OK_AND_ASSIGN( std::unique_ptr explicit_schema_read_context, @@ -2114,7 +2114,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { ASSERT_TRUE(explicit_schema_status.IsNotImplemented()) << explicit_schema_status.ToString(); ASSERT_NOK_WITH_MSG(explicit_schema_status, "An explicit read schema is not supported on a query-auth table"); - TableLoadOptions search_read_context_load_62{path}; + TableLoadContext search_read_context_load_62{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr search_read_context, ReadContextBuilder().EnablePreciseFileIndexEvaluate(true).Finish()); @@ -2123,7 +2123,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { "Search is not supported on a query-auth table"); ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - TableLoadOptions realtime_read_context_load_63{path}; + TableLoadContext realtime_read_context_load_63{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr realtime_read_context, ReadContextBuilder().WithRealtimeContext(realtime_context).Finish()); @@ -2133,14 +2133,14 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { const std::string row_id = SpecialFields::RowId().Name(); const std::string not_in_schema = fmt::format("NotImplemented: Column '{}' is not a column of the table schema", row_id); - TableLoadOptions row_id_read_context_load_64{path}; + TableLoadContext row_id_read_context_load_64{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr row_id_read_context, ReadContextBuilder().SetReadFieldNames({"id", row_id}).Finish()); ASSERT_NOK_WITH_MSG( TableRead::Create(row_id_read_context_load_64, std::move(row_id_read_context), {}), not_in_schema); - TableLoadOptions row_id_by_id_read_context_load_65{path}; + TableLoadContext row_id_by_id_read_context_load_65{path}; ASSERT_OK_AND_ASSIGN( std::unique_ptr row_id_by_id_read_context, @@ -2149,7 +2149,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { std::move(row_id_by_id_read_context), {}), fmt::format("NotImplemented: Field {} is not a column of the table schema", SpecialFields::RowId().Id())); - TableLoadOptions row_id_filter_read_context_load_66{path}; + TableLoadContext row_id_filter_read_context_load_66{path}; ASSERT_OK_AND_ASSIGN( std::unique_ptr row_id_filter_read_context, @@ -2161,7 +2161,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { ASSERT_NOK_WITH_MSG(TableRead::Create(row_id_filter_read_context_load_66, std::move(row_id_filter_read_context), {}), not_in_schema); - TableLoadOptions row_id_unfiltered_read_context_load_67{path}; + TableLoadContext row_id_unfiltered_read_context_load_67{path}; ASSERT_OK_AND_ASSIGN( std::unique_ptr row_id_unfiltered_read_context, @@ -2175,7 +2175,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { not_in_schema); ASSERT_OK_AND_ASSIGN(std::shared_ptr auth_catalog, CreateQueryAuthCatalog(*helper, dir->GetFileSystem())); - TableLoadOptions scan_context_load_68{path}; + TableLoadContext scan_context_load_68{path}; scan_context_load_68.catalog = auth_catalog; scan_context_load_68.identifier.emplace(identifier); @@ -2183,7 +2183,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { ScanContextBuilder().SetReadFieldNames({"id", row_id}).Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_68, std::move(scan_context), {}), not_in_schema); - TableLoadOptions scan_context_load_69{path}; + TableLoadContext scan_context_load_69{path}; scan_context_load_69.catalog = auth_catalog; scan_context_load_69.identifier.emplace(identifier); @@ -2193,7 +2193,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { .Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_69, std::move(scan_context), {}), not_in_schema); - TableLoadOptions scan_context_load_70{path}; + TableLoadContext scan_context_load_70{path}; scan_context_load_70.catalog = auth_catalog; scan_context_load_70.identifier.emplace(identifier); @@ -2201,7 +2201,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { ScanContextBuilder().SetPartitionFilter({{{"gone", "1"}}}).Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_70, std::move(scan_context), {}), "NotImplemented: Column 'gone' is not a column of the table schema"); - TableLoadOptions scan_context_load_71{path}; + TableLoadContext scan_context_load_71{path}; scan_context_load_71.catalog = auth_catalog; scan_context_load_71.identifier.emplace(identifier); @@ -2211,7 +2211,7 @@ TEST(TableTest, TestQueryAuthTableRefusesWhatCannotApplyItsRules) { .Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_71, std::move(scan_context), {}), "Search is not supported on a query-auth table"); - TableLoadOptions scan_context_load_72{path}; + TableLoadContext scan_context_load_72{path}; scan_context_load_72.catalog = auth_catalog; scan_context_load_72.identifier.emplace(identifier); @@ -2256,7 +2256,7 @@ TEST(TableTest, TestQueryAuthRefusesFallbackReadsAcrossBranches) { .CreateTable(QueryAuthTableSchema(), /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::QUERY_AUTH_ENABLED, enabled}}) .status()); - TableLoadOptions context_load_73{path}; + TableLoadContext context_load_73{path}; std::map context_options_73; context_options_73[Options::SCAN_FALLBACK_BRANCH] = branch; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); @@ -2275,7 +2275,7 @@ TEST(TableTest, TestQueryAuthRefusesFallbackReadsAcrossBranches) { .CreateTable(fallback_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{{Options::QUERY_AUTH_ENABLED, "true"}}) .status()); - TableLoadOptions fallback_read_context_load_74{fallback_path}; + TableLoadContext fallback_read_context_load_74{fallback_path}; std::map fallback_read_context_options_74; fallback_read_context_options_74[Options::SCAN_FALLBACK_BRANCH] = "fallback"; ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_read_context, @@ -2328,7 +2328,7 @@ TEST(TableTest, TestQueryAuthSystemTablesValidateTheBranchRead) { ASSERT_NOK_WITH_MSG( SystemTableLoader::LoadFromPath(dir->GetFileSystem(), path + "$branch_dev$" + name, {}), refused); - TableLoadOptions scan_context_load_75{system_path}; + TableLoadContext scan_context_load_75{system_path}; std::map scan_context_options_75; scan_context_options_75[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, @@ -2336,7 +2336,7 @@ TEST(TableTest, TestQueryAuthSystemTablesValidateTheBranchRead) { ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_75, std::move(scan_context), scan_context_options_75), refused); - TableLoadOptions read_context_load_76{system_path}; + TableLoadContext read_context_load_76{system_path}; std::map read_context_options_76; read_context_options_76[Options::BRANCH] = "dev"; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, @@ -2346,14 +2346,14 @@ TEST(TableTest, TestQueryAuthSystemTablesValidateTheBranchRead) { refused); const std::string restricted_path = path + "$branch_restricted$" + name; - TableLoadOptions scan_context_load_77{restricted_path}; + TableLoadContext scan_context_load_77{restricted_path}; scan_context_load_77.catalog = catalog; scan_context_load_77.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(scan_context_load_77, std::move(scan_context), {}), refused); - TableLoadOptions read_context_load_78{restricted_path}; + TableLoadContext read_context_load_78{restricted_path}; read_context_load_78.catalog = catalog; read_context_load_78.identifier.emplace(identifier); @@ -2363,13 +2363,13 @@ TEST(TableTest, TestQueryAuthSystemTablesValidateTheBranchRead) { ASSERT_EQ(Identifier("foo", "bar$branch_restricted"), catalog->LoadTableSchemaIdentifiers().back()); } - TableLoadOptions main_read_context_load_79{path + "$files"}; + TableLoadContext main_read_context_load_79{path + "$files"}; ASSERT_OK_AND_ASSIGN(std::unique_ptr main_read_context, ReadContextBuilder().Finish()); ASSERT_OK( TableRead::Create(main_read_context_load_79, std::move(main_read_context), {}).status()); - TableLoadOptions main_scan_context_load_80{path + "$files"}; + TableLoadContext main_scan_context_load_80{path + "$files"}; main_scan_context_load_80.catalog = catalog; main_scan_context_load_80.identifier.emplace(identifier); @@ -2377,7 +2377,7 @@ TEST(TableTest, TestQueryAuthSystemTablesValidateTheBranchRead) { ScanContextBuilder().Finish()); ASSERT_OK( TableScan::Create(main_scan_context_load_80, std::move(main_scan_context), {}).status()); - TableLoadOptions main_read_context_load_81{path + "$files"}; + TableLoadContext main_read_context_load_81{path + "$files"}; main_read_context_load_81.catalog = catalog; main_read_context_load_81.identifier.emplace(identifier); @@ -2462,45 +2462,45 @@ TEST(TableTest, TestCreateBuildsTheOperator) { // `TableScan::Create()` and friends build the operator straight from the context, which // carries the catalog and identifier it resolves the table through. - TableLoadOptions scan_ctx_builder_load_options{dir->Str()}; + TableLoadContext scan_ctx_builder_load_context{dir->Str()}; std::map scan_ctx_builder_options; ScanContextBuilder scan_ctx_builder; - scan_ctx_builder_load_options.catalog = catalog; - scan_ctx_builder_load_options.identifier.emplace(identifier); + scan_ctx_builder_load_context.catalog = catalog; + scan_ctx_builder_load_context.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, - TableScan::Create(scan_ctx_builder_load_options, std::move(scan_ctx), + TableScan::Create(scan_ctx_builder_load_context, std::move(scan_ctx), scan_ctx_builder_options)); - TableLoadOptions read_ctx_builder_load_options{dir->Str()}; + TableLoadContext read_ctx_builder_load_context{dir->Str()}; std::map read_ctx_builder_options; ReadContextBuilder read_ctx_builder; - read_ctx_builder_load_options.catalog = catalog; - read_ctx_builder_load_options.identifier.emplace(identifier); + read_ctx_builder_load_context.catalog = catalog; + read_ctx_builder_load_context.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, - TableRead::Create(read_ctx_builder_load_options, std::move(read_ctx), + TableRead::Create(read_ctx_builder_load_context, std::move(read_ctx), read_ctx_builder_options)); - TableLoadOptions write_ctx_builder_load_options{dir->Str()}; + TableLoadContext write_ctx_builder_load_context{dir->Str()}; std::map write_ctx_builder_options; WriteContextBuilder write_ctx_builder("commit-user"); - write_ctx_builder_load_options.catalog = catalog; - write_ctx_builder_load_options.identifier.emplace(identifier); + write_ctx_builder_load_context.catalog = catalog; + write_ctx_builder_load_context.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_ctx, write_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, - FileStoreWrite::Create(write_ctx_builder_load_options, + FileStoreWrite::Create(write_ctx_builder_load_context, std::move(write_ctx), write_ctx_builder_options)); - TableLoadOptions commit_ctx_builder_load_options{dir->Str()}; + TableLoadContext commit_ctx_builder_load_context{dir->Str()}; std::map commit_ctx_builder_options; CommitContextBuilder commit_ctx_builder("commit-user"); - commit_ctx_builder_load_options.catalog = catalog; - commit_ctx_builder_load_options.identifier.emplace(identifier); + commit_ctx_builder_load_context.catalog = catalog; + commit_ctx_builder_load_context.identifier.emplace(identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_ctx, commit_ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN( [[maybe_unused]] std::unique_ptr commit, - FileStoreCommit::Create(commit_ctx_builder_load_options, std::move(commit_ctx), + FileStoreCommit::Create(commit_ctx_builder_load_context, std::move(commit_ctx), commit_ctx_builder_options)); } @@ -2732,7 +2732,7 @@ TEST(TableTest, TestOperatorFactoryPreservesLoadedSchemaCache) { ASSERT_TRUE(dir); ASSERT_OK(CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); std::shared_ptr
table; - ASSERT_OK_AND_ASSIGN(table, TableFactory::Create(TableLoadOptions{dir->Str()}, {}, + ASSERT_OK_AND_ASSIGN(table, TableFactory::Create(TableLoadContext{dir->Str()}, {}, commit ? "commit" : "write")); auto managed = checked_pointer_cast(table); ASSERT_OK(dir->GetFileSystem()->Delete(dir->Str() + "/schema/schema-0", false)); @@ -2761,7 +2761,7 @@ TEST(TableTest, TestMainBranchAliasesUseMainSchemaForAllOperatorEntryPoints) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(identifier)); for (bool static_factory : {false, true}) { SCOPED_TRACE(alias + (static_factory ? ":static" : ":loaded")); - TableLoadOptions builder_load{path}; + TableLoadContext builder_load{path}; std::map builder_options; ScanContextBuilder builder; builder_load.catalog = catalog; @@ -2779,7 +2779,7 @@ TEST(TableTest, TestMainBranchAliasesUseMainSchemaForAllOperatorEntryPoints) { ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_TRUE(plan->Splits().empty()); - TableLoadOptions read_builder_load{path}; + TableLoadContext read_builder_load{path}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_load.catalog = catalog; @@ -2796,7 +2796,7 @@ TEST(TableTest, TestMainBranchAliasesUseMainSchemaForAllOperatorEntryPoints) { ASSERT_OK_AND_ASSIGN(read, configured_table_20->NewRead(std::move(read_context))); } - TableLoadOptions write_builder_load{path}; + TableLoadContext write_builder_load{path}; std::map write_builder_options; WriteContextBuilder write_builder("user"); write_builder_load.catalog = catalog; @@ -2816,7 +2816,7 @@ TEST(TableTest, TestMainBranchAliasesUseMainSchemaForAllOperatorEntryPoints) { } ASSERT_OK(write->Close()); - TableLoadOptions commit_builder_load{path}; + TableLoadContext commit_builder_load{path}; std::map commit_builder_options; CommitContextBuilder commit_builder("user"); commit_builder_load.catalog = catalog; @@ -2914,28 +2914,28 @@ TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { std::map{{Options::BRANCH, "main"}}}) { const std::string branch = overrides.empty() ? "rt" : overrides.at(Options::BRANCH); SCOPED_TRACE(branch); - TableLoadOptions scan_context_builder_30072_load_options{path}; + TableLoadContext scan_context_builder_30072_load_context{path}; std::map scan_context_builder_30072_options; ScanContextBuilder scan_context_builder_30072; - scan_context_builder_30072_load_options.table_schema_json = schema_json; + scan_context_builder_30072_load_context.table_schema_json = schema_json; scan_context_builder_30072_options = overrides; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder_30072.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(scan_context_builder_30072_load_options, std::move(scan_context), + TableScan::Create(scan_context_builder_30072_load_context, std::move(scan_context), scan_context_builder_30072_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(branch == "rt" ? 1 : 2, plan->SnapshotId()); - TableLoadOptions read_context_builder_34363_load_options{path}; + TableLoadContext read_context_builder_34363_load_context{path}; std::map read_context_builder_34363_options; ReadContextBuilder read_context_builder_34363; - read_context_builder_34363_load_options.table_schema_json = schema_json; + read_context_builder_34363_load_context.table_schema_json = schema_json; read_context_builder_34363_options = overrides; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder_34363.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, - TableFactory::Create(read_context_builder_34363_load_options, + TableFactory::Create(read_context_builder_34363_load_context, read_context_builder_34363_options, "read")); ASSERT_EQ(branch, std::dynamic_pointer_cast(loaded)->CurrentBranch()); ASSERT_EQ(BranchManager::BranchPath(path, branch), @@ -2959,15 +2959,15 @@ TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { ASSERT_TRUE(snapshot); ASSERT_EQ(snapshot->TotalRecordCount(), rows->length()); } - TableLoadOptions explicit_main_builder_load_options{path}; + TableLoadContext explicit_main_builder_load_context{path}; std::map explicit_main_builder_options; ReadContextBuilder explicit_main_builder; - explicit_main_builder_load_options.table_schema_json = schema_json; + explicit_main_builder_load_context.table_schema_json = schema_json; explicit_main_builder_options["branch"] = "main"; ASSERT_OK_AND_ASSIGN(std::unique_ptr explicit_main, explicit_main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, - TableFactory::Create(explicit_main_builder_load_options, + TableFactory::Create(explicit_main_builder_load_context, explicit_main_builder_options, "read")); ASSERT_EQ("main", std::dynamic_pointer_cast(loaded)->CurrentBranch()); } @@ -3001,10 +3001,10 @@ TEST(TableTest, TestLoadedTableReadRetainsLoadSchemeMap) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db", "tbl"))); for (bool direct : {false, true}) { SCOPED_TRACE(direct); - TableLoadOptions context_builder_36442_load_options{path}; + TableLoadContext context_builder_36442_load_context{path}; std::map context_builder_36442_options; ReadContextBuilder context_builder_36442; - context_builder_36442_load_options.fs_scheme_to_identifier_map = scheme_map; + context_builder_36442_load_context.fs_scheme_to_identifier_map = scheme_map; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_36442.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, @@ -3013,7 +3013,7 @@ TEST(TableTest, TestLoadedTableReadRetainsLoadSchemeMap) { table->Copy(context_builder_36442_options)); return configured->NewRead(std::move(context)); })() - : TableRead::Create(context_builder_36442_load_options, std::move(context), + : TableRead::Create(context_builder_36442_load_context, std::move(context), context_builder_36442_options)); auto* append_read = dynamic_cast(read.get()); ASSERT_NE(nullptr, append_read); @@ -3095,33 +3095,33 @@ TEST(TableTest, TestSystemTablePathRetainsSchemeMapForDependencyReads) { const std::map scheme_map = {{"file", "local"}}; for (bool reload_context : {false, true}) { SCOPED_TRACE(reload_context); - TableLoadOptions context_builder_41670_load_options{dir->Str() + "$ro"}; + TableLoadContext context_builder_41670_load_context{dir->Str() + "$ro"}; std::map context_builder_41670_options; ReadContextBuilder context_builder_41670; - context_builder_41670_load_options.fs_scheme_to_identifier_map = scheme_map; + context_builder_41670_load_context.fs_scheme_to_identifier_map = scheme_map; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_41670.Finish()); std::unique_ptr read; if (reload_context) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - TableFactory::Create(context_builder_41670_load_options, + TableFactory::Create(context_builder_41670_load_context, context_builder_41670_options, "read")); - TableLoadOptions context_builder_42682_load_options{dir->Str()}; + TableLoadContext context_builder_42682_load_context{dir->Str()}; std::map context_builder_42682_options; ReadContextBuilder context_builder_42682; - context_builder_41670_load_options.path = context_builder_42682_load_options.path; - context_builder_41670_load_options.file_system = - context_builder_42682_load_options.file_system; - context_builder_41670_load_options.fs_scheme_to_identifier_map = - context_builder_42682_load_options.fs_scheme_to_identifier_map; - context_builder_41670_load_options.table_schema_json = - context_builder_42682_load_options.table_schema_json; - context_builder_41670_load_options.catalog = context_builder_42682_load_options.catalog; - context_builder_41670_load_options.table_id = - context_builder_42682_load_options.table_id; - context_builder_41670_load_options.identifier.reset(); - if (context_builder_42682_load_options.identifier) { - context_builder_41670_load_options.identifier.emplace( - *context_builder_42682_load_options.identifier); + context_builder_41670_load_context.path = context_builder_42682_load_context.path; + context_builder_41670_load_context.file_system = + context_builder_42682_load_context.file_system; + context_builder_41670_load_context.fs_scheme_to_identifier_map = + context_builder_42682_load_context.fs_scheme_to_identifier_map; + context_builder_41670_load_context.table_schema_json = + context_builder_42682_load_context.table_schema_json; + context_builder_41670_load_context.catalog = context_builder_42682_load_context.catalog; + context_builder_41670_load_context.table_id = + context_builder_42682_load_context.table_id; + context_builder_41670_load_context.identifier.reset(); + if (context_builder_42682_load_context.identifier) { + context_builder_41670_load_context.identifier.emplace( + *context_builder_42682_load_context.identifier); } context_builder_41670_options = context_builder_42682_options; ASSERT_OK_AND_ASSIGN(context, context_builder_42682.Finish()); @@ -3130,7 +3130,7 @@ TEST(TableTest, TestSystemTablePathRetainsSchemeMapForDependencyReads) { ASSERT_OK_AND_ASSIGN(read, configured_table_29->NewRead(std::move(context))); } else { ASSERT_OK_AND_ASSIGN( - read, TableRead::Create(context_builder_41670_load_options, std::move(context), + read, TableRead::Create(context_builder_41670_load_context, std::move(context), context_builder_41670_options)); } auto* append_read = dynamic_cast(read.get()); @@ -3230,7 +3230,7 @@ TEST(TableTest, SystemTablePathUsesEffectiveBranchSchema) { const std::map options = path_branch ? std::map() : std::map{{Options::BRANCH, "dev"}}; - TableLoadOptions context_load_90{path}; + TableLoadContext context_load_90{path}; std::map context_options_90; context_options_90 = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, @@ -3244,7 +3244,7 @@ TEST(TableTest, SystemTablePathUsesEffectiveBranchSchema) { ASSERT_EQ(dev_schema->field_names(), append_read->context_->GetTableSchema()->FieldNames()); ASSERT_EQ((std::vector{"extra"}), append_read->context_->GetReadSchema()->field_names()); - TableLoadOptions scan_context_load_91{path}; + TableLoadContext scan_context_load_91{path}; std::map scan_context_options_91; scan_context_options_91 = options; ASSERT_OK_AND_ASSIGN( @@ -3312,7 +3312,7 @@ TEST(TableTest, StaticFallbackReadUsesSuppliedMainSchema) { CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl", "dev"), "dev")); ASSERT_OK_AND_ASSIGN(std::string main_json, main->GetSchema()->GetJsonSchema()); ASSERT_OK(fs->Delete(dir->Str() + "/schema", true)); - TableLoadOptions context_load_94{dir->Str()}; + TableLoadContext context_load_94{dir->Str()}; std::map context_options_94; context_options_94[Options::BRANCH] = "dev"; context_options_94[Options::SCAN_FALLBACK_BRANCH] = "main"; @@ -3328,7 +3328,7 @@ TEST(TableTest, StaticFallbackReadUsesSuppliedMainSchema) { ASSERT_EQ(main->GetSchema()->FieldNames(), fallback_read->context_->GetTableSchema()->FieldNames()); ASSERT_EQ("main", fallback_read->context_->GetCoreOptions().GetBranch()); - TableLoadOptions missing_schema_load_95{dir->Str()}; + TableLoadContext missing_schema_load_95{dir->Str()}; std::map missing_schema_options_95; missing_schema_options_95[Options::BRANCH] = "dev"; missing_schema_options_95[Options::SCAN_FALLBACK_BRANCH] = "main"; @@ -3503,7 +3503,7 @@ TEST(TableTest, StaticReadOptimizedPreservesSuppliedSchemaAndMandatoryOptions) { arrow::field("extra", arrow::int32())}), {}, {}, {})); ASSERT_OK_AND_ASSIGN(std::string json, supplied->GetJsonSchema()); - TableLoadOptions context_load_104{dir->Str() + "$ro"}; + TableLoadContext context_load_104{dir->Str() + "$ro"}; context_load_104.table_schema_json = json; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, @@ -3514,7 +3514,7 @@ TEST(TableTest, StaticReadOptimizedPreservesSuppliedSchemaAndMandatoryOptions) { ASSERT_NE(nullptr, append); ASSERT_EQ((std::vector{"extra"}), append->context_->GetReadSchema()->field_names()); - TableLoadOptions scan_context_load_105{dir->Str() + "$ro"}; + TableLoadContext scan_context_load_105{dir->Str() + "$ro"}; std::map scan_context_options_105; scan_context_load_105.table_schema_json = json; scan_context_options_105[kReadOptimizedScanOption] = "false"; @@ -3531,7 +3531,7 @@ TEST(TableTest, StaticReadOptimizedPreservesSuppliedSchemaAndMandatoryOptions) { SchemaManager manager(dir->GetFileSystem(), pk_path); ASSERT_OK(manager.CreateTable(arrow::schema({arrow::field("id", arrow::int32(), false)}), {}, {"id"}, {{Options::BUCKET, "1"}})); - TableLoadOptions scan_context_load_106{pk_path + "$ro"}; + TableLoadContext scan_context_load_106{pk_path + "$ro"}; std::map scan_context_options_106; scan_context_options_106[kReadOptimizedScanOption] = "false"; ASSERT_OK_AND_ASSIGN(scan_context, ScanContextBuilder().WithStreamingMode(true).Finish()); @@ -3547,13 +3547,13 @@ TEST(TableTest, StaticSystemTablePathPreservesCaseSensitiveBranch) { const std::string branch = "MAIN"; ASSERT_OK(CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"), branch)); const std::string path = dir->Str() + "$branch_MAIN$schemas"; - TableLoadOptions scan_context_load_107{path}; + TableLoadContext scan_context_load_107{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(scan_context_load_107, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - TableLoadOptions read_context_load_108{path}; + TableLoadContext read_context_load_108{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, @@ -3574,7 +3574,7 @@ TEST(TableTest, StaticFallbackResolvesStorageFromFallbackOptions) { ASSERT_OK(main.CreateTable(schema, {}, {}, {{"test.endpoint", "fallback"}})); ASSERT_OK(dev.CreateTable(schema, {}, {}, {{"test.endpoint", "primary"}})); for (bool explicit_fs : {false, true}) { - TableLoadOptions builder_load{dir->Str()}; + TableLoadContext builder_load{dir->Str()}; std::map builder_options; ReadContextBuilder builder; builder_options[Options::BRANCH] = "dev"; @@ -3604,9 +3604,9 @@ TEST(TableTest, StaticFallbackResolvesStorageFromFallbackOptions) { namespace paimon::test { -TEST(TableTest, TestLoadOptionsValidateIdentityBeforeMetadataAccess) { +TEST(TableTest, TestLoadContextValidateIdentityBeforeMetadataAccess) { auto catalog = std::make_shared(); - TableLoadOptions load{"unused/table"}; + TableLoadContext load{"unused/table"}; load.identifier.emplace("db", "tbl"); ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "read"), "non-null catalog"); load.catalog = catalog; @@ -3620,10 +3620,10 @@ TEST(TableTest, TestLoadOptionsValidateIdentityBeforeMetadataAccess) { ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); } -TEST(TableTest, TestLoadOptionsRejectConflictingFileSystemResolution) { +TEST(TableTest, TestLoadContextRejectConflictingFileSystemResolution) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - TableLoadOptions load{dir->Str()}; + TableLoadContext load{dir->Str()}; load.file_system = dir->GetFileSystem(); load.fs_scheme_to_identifier_map = {{"file", "local"}}; ASSERT_NOK_WITH_MSG(TableFactory::Create(load, {}, "read"), "cannot be specified together"); @@ -3637,7 +3637,7 @@ TEST(TableTest, TestPreloadedSchemaIsIndependentOfReadProjection) { arrow::schema({arrow::field("id", arrow::int32()), arrow::field("value", arrow::utf8())}), {}, {}, {})); - TableLoadOptions load{dir->Str() + "/without-metadata"}; + TableLoadContext load{dir->Str() + "/without-metadata"}; load.file_system = dir->GetFileSystem(); ASSERT_OK_AND_ASSIGN(load.table_schema_json, schema->GetJsonSchema()); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, @@ -3678,7 +3678,7 @@ TEST(TableTest, TestCatalogOnlyLoadsRetainTheResolvedBranchSchema) { auto catalog = std::make_shared(); catalog->SetTable(original); catalog->SetTableLocation(path); - TableLoadOptions load; + TableLoadContext load; load.catalog = catalog; load.identifier.emplace(identifier); load.table_id = "uuid"; @@ -3711,7 +3711,7 @@ TEST(TableTest, TestCatalogOnlyLoadsRetainTheResolvedBranchSchema) { } } -TEST(TableTest, TestLoadOptionsValidateExpectedCatalogUuid) { +TEST(TableTest, TestLoadContextValidateExpectedCatalogUuid) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN( @@ -3720,7 +3720,7 @@ TEST(TableTest, TestLoadOptionsValidateExpectedCatalogUuid) { auto catalog = std::make_shared(true, "current-uuid"); catalog->SetTableSchema(schema); catalog->SetFileSystem(dir->GetFileSystem()); - TableLoadOptions load{dir->Str()}; + TableLoadContext load{dir->Str()}; load.catalog = catalog; load.identifier.emplace("db", "tbl"); load.table_id = "previous-uuid"; @@ -3760,7 +3760,7 @@ TEST(TableTest, TestExplicitPathAndUuidRetainCatalogFileSystemWithRouting) { catalog->SetTableLocation(dir->Str()); for (bool override_file_system : {false, true}) { auto supplied_fs = std::make_shared(); - TableLoadOptions load{dir->Str()}; + TableLoadContext load{dir->Str()}; load.catalog = catalog; load.identifier.emplace("db", "tbl"); load.table_id = "table-uuid"; @@ -3786,7 +3786,7 @@ TEST(TableTest, TestMissingSchemaErrorsDistinguishCatalogAndPathLoads) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); for (bool with_catalog : {false, true}) { - TableLoadOptions load{dir->Str() + "/db.db/missing-table"}; + TableLoadContext load{dir->Str() + "/db.db/missing-table"}; if (with_catalog) { load.catalog = std::make_shared( dir->GetFileSystem(), dir->Str(), std::map{}); diff --git a/src/paimon/core/utils/blob_view_lookup.cpp b/src/paimon/core/utils/blob_view_lookup.cpp index 0e02439b8..f076c9863 100644 --- a/src/paimon/core/utils/blob_view_lookup.cpp +++ b/src/paimon/core/utils/blob_view_lookup.cpp @@ -28,7 +28,7 @@ #include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/executor/future.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -155,31 +155,31 @@ Result BlobViewLookup::LoadTableDescriptorChu Catalog::Create(catalog_context->root_path, catalog_context->options, /*file_system=*/nullptr, catalog_context->fs_scheme_to_identifier_map)); PAIMON_ASSIGN_OR_RAISE(std::string table_path, catalog->GetTableLocation(identifier)); - TableLoadOptions scan_builder_load_options{table_path}; + TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; ScanContextBuilder scan_builder; auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); - scan_builder_load_options.catalog = catalog; - scan_builder_load_options.identifier.emplace(identifier); + scan_builder_load_context.catalog = catalog; + scan_builder_load_context.identifier.emplace(identifier); scan_builder.SetGlobalIndexResult(global_index_result).WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - TableLoadOptions read_builder_load_options{table_path}; + TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; ReadContextBuilder read_builder; std::vector read_field_ids = field_ids; read_field_ids.push_back(SpecialFieldIds::ROW_ID); read_builder_options[Options::BLOB_AS_DESCRIPTOR] = "true"; - read_builder_load_options.catalog = catalog; - read_builder_load_options.identifier.emplace(identifier); + read_builder_load_context.catalog = catalog; + read_builder_load_context.identifier.emplace(identifier); read_builder.SetReadFieldIds(read_field_ids).EnablePrefetch(true).WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index f3d48ca24..a8e5f6739 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -40,7 +40,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/table.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/catalog_options.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" @@ -1241,7 +1241,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_EQ(3, schema->Id()); ASSERT_EQ("b1", schema->Options().at(Options::BRANCH)); - TableLoadOptions load; + TableLoadContext load; load.catalog = catalog; load.identifier.emplace(branch_identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, @@ -1842,13 +1842,13 @@ TEST_F(RestCatalogTest, FileStoreCommitIsBuiltFromTheCatalog) { requests_before = state_->get_table_requests; } std::shared_ptr shared_catalog(std::move(rest_catalog)); - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; CommitContextBuilder builder("commit-user"); - builder_load_options.catalog = shared_catalog; - builder_load_options.identifier.emplace(Identifier("db1", "t1")); + builder_load_context.catalog = shared_catalog; + builder_load_context.identifier.emplace(Identifier("db1", "t1")); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_OK(FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + ASSERT_OK(FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); { std::lock_guard lock(state_->mutex); ASSERT_GT(state_->get_table_requests, requests_before); @@ -1910,7 +1910,7 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { ASSERT_EQ(1, state_->get_table_requests); } - TableLoadOptions scan_context_load_2{dir->Str()}; + TableLoadContext scan_context_load_2{dir->Str()}; scan_context_load_2.catalog = catalog; scan_context_load_2.identifier.emplace(identifier); @@ -1936,7 +1936,7 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { sent.GetSelect()); } - TableLoadOptions read_context_load_3{dir->Str()}; + TableLoadContext read_context_load_3{dir->Str()}; std::map read_context_options_3; read_context_options_3[Options::QUERY_AUTH_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); @@ -1953,7 +1953,7 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { ASSERT_EQ("***", checked_pointer_cast(rows->GetFieldByName("f1"))->GetString(0)); - TableLoadOptions disabling_read_context_load_4{dir->Str()}; + TableLoadContext disabling_read_context_load_4{dir->Str()}; std::map disabling_read_context_options_4; disabling_read_context_load_4.catalog = catalog; disabling_read_context_load_4.identifier.emplace(identifier); @@ -1965,7 +1965,7 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { disabling_read_context_options_4), "Invalid: Option 'query-auth.enabled' is not allowed to be disabled dynamically"); - TableLoadOptions files_scan_context_load_5{dir->Str() + "$files"}; + TableLoadContext files_scan_context_load_5{dir->Str() + "$files"}; files_scan_context_load_5.catalog = catalog; files_scan_context_load_5.identifier.emplace(Identifier("db1", "t1$files")); @@ -1975,7 +1975,7 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { TableScan::Create(files_scan_context_load_5, std::move(files_scan_context), {}), "NotImplemented: System table 'files' is not supported on a table with " "'query-auth.enabled=true'"); - TableLoadOptions options_scan_context_load_6{dir->Str() + "$options"}; + TableLoadContext options_scan_context_load_6{dir->Str() + "$options"}; options_scan_context_load_6.catalog = catalog; options_scan_context_load_6.identifier.emplace(Identifier("db1", "t1$options")); @@ -1989,7 +1989,7 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { const Identifier system_identifier("sys", "tables"); for (const std::string& suffix : {std::string(), std::string("$files")}) { SCOPED_TRACE(suffix); - TableLoadOptions mismatched_scan_context_load_7{dir->Str() + suffix}; + TableLoadContext mismatched_scan_context_load_7{dir->Str() + suffix}; mismatched_scan_context_load_7.catalog = catalog; mismatched_scan_context_load_7.identifier.emplace(system_identifier); @@ -1998,7 +1998,7 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { Status scan_status = TableScan::Create(mismatched_scan_context_load_7, std::move(mismatched_scan_context), {}) .status(); - TableLoadOptions mismatched_read_context_load_8{dir->Str() + suffix}; + TableLoadContext mismatched_read_context_load_8{dir->Str() + suffix}; mismatched_read_context_load_8.catalog = catalog; mismatched_read_context_load_8.identifier.emplace(system_identifier); @@ -2013,7 +2013,7 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { const std::string global_path = PathUtil::JoinPath(dir->Str(), "sys/catalog_options"); const Identifier global_identifier("sys", "catalog_options"); - TableLoadOptions global_scan_context_load_9{global_path}; + TableLoadContext global_scan_context_load_9{global_path}; std::map global_scan_context_options_9; global_scan_context_load_9.catalog = catalog; global_scan_context_load_9.identifier.emplace(global_identifier); @@ -2023,7 +2023,7 @@ TEST_F(RestCatalogTest, ReadThroughTheCatalogAppliesItsQueryAuth) { ASSERT_OK(TableScan::Create(global_scan_context_load_9, std::move(global_scan_context), global_scan_context_options_9) .status()); - TableLoadOptions global_read_context_load_10{global_path}; + TableLoadContext global_read_context_load_10{global_path}; std::map global_read_context_options_10; global_read_context_load_10.catalog = catalog; global_read_context_load_10.identifier.emplace(global_identifier); @@ -2130,7 +2130,7 @@ TEST_F(RestCatalogTest, QueryAuthRejectsColumnsReplacedOnlyInTheCatalog) { std::shared_ptr catalog(std::move(rest_catalog)); auto new_scan = [&](const std::vector& fields) -> Result> { - TableLoadOptions context_load_11{dir->Str()}; + TableLoadContext context_load_11{dir->Str()}; context_load_11.catalog = catalog; context_load_11.identifier.emplace(Identifier("db1", "t1")); @@ -2215,7 +2215,7 @@ TEST_F(RestCatalogTest, ScanOfABranchFollowsTheQueryAuthOfThatBranch) { } std::shared_ptr catalog(std::move(rest_catalog)); - TableLoadOptions scan_context_load_12{dir->Str()}; + TableLoadContext scan_context_load_12{dir->Str()}; std::map scan_context_options_12; scan_context_load_12.catalog = catalog; scan_context_load_12.identifier.emplace(Identifier("db1", "t1")); @@ -2232,7 +2232,7 @@ TEST_F(RestCatalogTest, ScanOfABranchFollowsTheQueryAuthOfThatBranch) { state_->auth_requests); } - TableLoadOptions read_context_load_13{dir->Str()}; + TableLoadContext read_context_load_13{dir->Str()}; std::map read_context_options_13; read_context_load_13.catalog = catalog; read_context_load_13.identifier.emplace(Identifier("db1", "t1")); @@ -2283,7 +2283,7 @@ TEST_F(RestCatalogTest, ReadRejectsAFallbackBranchProtectedOnlyByTheCatalog) { } } std::shared_ptr catalog(std::move(rest_catalog)); - TableLoadOptions read_context_load_14{dir->Str()}; + TableLoadContext read_context_load_14{dir->Str()}; std::map read_context_options_14; read_context_load_14.catalog = catalog; read_context_load_14.identifier.emplace(Identifier("db1", "t1")); @@ -2296,7 +2296,7 @@ TEST_F(RestCatalogTest, ReadRejectsAFallbackBranchProtectedOnlyByTheCatalog) { for (const auto& [branch, message] : std::vector>{ {"MAIN", "a catalog names branch 'MAIN' as it names the main branch"}, {"dev$options", "a branch a catalog addresses cannot contain '$'"}}) { - TableLoadOptions read_context_load_15{dir->Str()}; + TableLoadContext read_context_load_15{dir->Str()}; std::map read_context_options_15; read_context_load_15.catalog = catalog; read_context_load_15.identifier.emplace(Identifier("db1", "t1")); @@ -2616,13 +2616,13 @@ TEST_P(RestCatalogCommitRecoveryTest, CommitAcceptedThenLostRecoversAfterRestart std::shared_ptr catalog(std::move(rest_catalog)); auto create_commit = [&]() -> Result> { - TableLoadOptions builder_load_options{dir->Str()}; + TableLoadContext builder_load_context{dir->Str()}; std::map builder_options; CommitContextBuilder builder("commit-user"); - builder_load_options.catalog = catalog; - builder_load_options.identifier.emplace(identifier); + builder_load_context.catalog = catalog; + builder_load_context.identifier.emplace(identifier); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(builder_load_options, std::move(context), builder_options); + return FileStoreCommit::Create(builder_load_context, std::move(context), builder_options); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, create_commit()); @@ -2794,20 +2794,20 @@ class RestCatalogCommitLeaseTest : public RestCatalogTest { Result> CreateCommit( const std::shared_ptr& catalog, const std::optional& table_id, const std::map& options) { - TableLoadOptions builder_load_options{dir_->Str()}; + TableLoadContext builder_load_context{dir_->Str()}; std::map builder_options; CommitContextBuilder builder("commit-user"); - builder_load_options.catalog = catalog; - builder_load_options.identifier.emplace(Identifier("db1", "t1")); + builder_load_context.catalog = catalog; + builder_load_context.identifier.emplace(Identifier("db1", "t1")); if (table_id) { - builder_load_options.table_id = table_id.value(); + builder_load_context.table_id = table_id.value(); } for (const auto& [key, value] : options) { builder_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreCommit::Create(builder_load_options, std::move(context), builder_options); + return FileStoreCommit::Create(builder_load_context, std::move(context), builder_options); } std::vector TableEvents() { diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index 87bf02654..f6859b3fe 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -32,7 +32,7 @@ #include "fmt/format.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" @@ -91,7 +91,7 @@ class TestHelper { PAIMON_ASSIGN_OR_RAISE(auto file_system, FileSystemFactory::Get(file_system_identifier, table_path, options)); std::string commit_user = "commit_user"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder(commit_user); if (!temp_directory.empty()) { @@ -103,9 +103,9 @@ class TestHelper { context_builder.WithStreamingMode(is_streaming_mode).WithWriteId(12345).Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr write, - FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(commit_user); commit_context_builder_options = options; @@ -113,7 +113,7 @@ class TestHelper { commit_context_builder.IgnoreEmptyCommit(false).Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); return std::unique_ptr(new TestHelper(std::move(file_system), std::move(write), std::move(commit), commit_user, @@ -164,7 +164,7 @@ class TestHelper { Result>> NewScan(StartupMode startup_mode, std::optional snapshot_id, bool is_streaming = true) { - TableLoadOptions scan_context_builder_load_options{table_path_}; + TableLoadContext scan_context_builder_load_context{table_path_}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options_; @@ -176,7 +176,7 @@ class TestHelper { } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( - scan_, TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + scan_, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); return Scan(); } @@ -288,7 +288,7 @@ class TestHelper { Result> ReadResult( const std::vector>& splits, std::unique_ptr<::ArrowSchema> read_schema) { - TableLoadOptions read_context_builder_load_options{table_path_}; + TableLoadContext read_context_builder_load_context{table_path_}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options_; @@ -298,7 +298,7 @@ class TestHelper { } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); @@ -319,14 +319,14 @@ class TestHelper { Result ReadAndCheckResult(const std::shared_ptr& data_type, const std::vector>& splits, const std::string& expected_result) { - TableLoadOptions read_context_builder_load_options{table_path_}; + TableLoadContext read_context_builder_load_context{table_path_}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 6b420a6e9..32541ca09 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -24,7 +24,7 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" @@ -416,14 +416,14 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMap auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); - TableLoadOptions read_context_builder_load_options{ + TableLoadContext read_context_builder_load_context{ PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); @@ -1045,7 +1045,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableCompactionDictionaryPassthrough) auto predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"s", FieldType::STRING, Literal(FieldType::STRING, indexed_value.data(), indexed_value.size())); - TableLoadOptions read_context_builder_load_options{ + TableLoadContext read_context_builder_load_context{ PathUtil::JoinPath(dir->Str(), "foo.db/bar")}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; @@ -1053,7 +1053,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableCompactionDictionaryPassthrough) read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto filtered, ReadResultCollector::CollectResult(batch_reader.get())); diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index f2031fdbe..1cf069573 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -40,7 +40,7 @@ #include "arrow/ipc/json_simple.h" #include "arrow/type.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" @@ -142,14 +142,14 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter const std::vector& write_cols, const std::vector>& write_arrays) const { // write - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); for (const auto& write_array : write_arrays) { @@ -181,14 +181,14 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_store_commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -276,7 +276,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter Result> ScanTable(const std::string& table_path, const std::shared_ptr& predicate = nullptr, const std::vector& row_ranges = {}) const { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetPredicate(predicate); @@ -285,7 +285,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter scan_context_builder.SetGlobalIndexResult(global_index_result); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); @@ -298,7 +298,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter const std::shared_ptr& plan, const std::shared_ptr& predicate = nullptr, const std::map& options = {}) const { auto splits = plan->Splits(); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); @@ -308,7 +308,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); @@ -4913,14 +4913,14 @@ TEST_P(BlobTableInteTest, TestReadArrayBlobTableFromJava) { const std::vector read_fields = {"id", "array_payloads"}; for (int64_t snapshot_id : {1, 2, 3}) { - TableLoadOptions scan_builder_load_options{table_path}; + TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); @@ -4979,14 +4979,14 @@ TEST_P(BlobTableInteTest, TestReadMapBlobTableFromJava) { "large_decimal_payloads"}; for (int64_t snapshot_id : {1, 3}) { - TableLoadOptions scan_builder_load_options{table_path}; + TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index 4a32b7d48..1fe6a7612 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -42,7 +42,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -205,7 +205,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { auto dir = UniqueTestDirectory::Create(); std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options = clean_options; @@ -213,7 +213,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( auto commit, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); @@ -225,7 +225,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1"))); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options = clean_options; @@ -233,7 +233,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( auto commit, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); @@ -245,7 +245,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath( table_path, "f1=10/bucket-1/data-7a912f84-04b7-4bbb-8dc6-53f4a292ea25-0.parquet"))); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options = clean_options; @@ -253,7 +253,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( auto commit, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 4); @@ -264,7 +264,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { std::string table_path = dir->Str(); ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path)); ASSERT_OK(file_system_->Delete(PathUtil::JoinPath(table_path, "snapshot/snapshot-3"))); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options = clean_options; @@ -272,7 +272,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) { commit_context_builder.IgnoreEmptyCommit(false).Finish()); ASSERT_OK_AND_ASSIGN( auto commit, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK_AND_ASSIGN(int32_t expired_items, commit->Expire()); ASSERT_EQ(expired_items, 1); @@ -303,7 +303,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { CreateTestTable(dir->Str(), /*db_name=*/"foo", /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; @@ -311,7 +311,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, @@ -328,11 +328,11 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; - commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_load_context.file_system = file_system_; commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; @@ -343,7 +343,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { .IgnoreEmptyCommit(false) .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0, 10)); @@ -461,7 +461,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { /*table_name=*/"bar", &arrow_schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; @@ -470,7 +470,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { ASSERT_OK_AND_ASSIGN( std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, @@ -487,11 +487,11 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; - commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_load_context.file_system = file_system_; commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MAX] = "2"; commit_context_builder_options[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1"; commit_context_builder_options[Options::SNAPSHOT_TIME_RETAINED] = "1ms"; @@ -504,7 +504,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { .Finish()); ASSERT_OK_AND_ASSIGN( auto commit, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); @@ -630,7 +630,7 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder(commit_user); context_builder_options = options; @@ -638,7 +638,7 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, @@ -655,14 +655,14 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(commit_user); commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; - commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_load_context.file_system = file_system_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); @@ -751,7 +751,7 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { /*partition_keys=*/{"f1", "f2"}, /*primary_keys=*/{}, options)); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder(commit_user); context_builder_options = options; @@ -759,7 +759,7 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, @@ -776,14 +776,14 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(commit_user); commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; - commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_load_context.file_system = file_system_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.IgnoreEmptyCommit(false).Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); @@ -868,15 +868,15 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder(commit_user); context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.WithStreamingMode(true).Finish()); - std::string root_path = context_builder_load_options.path; + std::string root_path = context_builder_load_context.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, @@ -893,18 +893,18 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit(/*wait_compaction=*/false, 0)); ASSERT_EQ(3u, results.size()); - TableLoadOptions commit_context_builder_load_options{root_path}; + TableLoadContext commit_context_builder_load_context{root_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder(commit_user); commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; - commit_context_builder_load_options.file_system = file_system_; + commit_context_builder_load_context.file_system = file_system_; commit_context_builder_options[Options::BUCKET] = "2"; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder .IgnoreEmptyCommit(false) .Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK(commit->Commit(results, 0)); diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 4718d92a1..a65258832 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -24,7 +24,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -112,7 +112,7 @@ class DataEvolutionTableTest : public ::testing::Test, const std::vector>& write_arrays, const std::map& write_options = {}) const { // write - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithWriteSchema(write_cols); @@ -122,7 +122,7 @@ class DataEvolutionTableTest : public ::testing::Test, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); for (const auto& write_array : write_arrays) { ArrowArray c_array; @@ -170,14 +170,14 @@ class DataEvolutionTableTest : public ::testing::Test, Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_store_commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -317,7 +317,7 @@ class DataEvolutionTableTest : public ::testing::Test, const std::string& table_path, const std::vector& read_schema, int32_t limit, const std::shared_ptr& predicate = nullptr, const std::vector& row_ranges = {}, bool enable_predicate_filter = false) const { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(limit); @@ -331,13 +331,13 @@ class DataEvolutionTableTest : public ::testing::Test, FinishScanContext(scan_context_builder, scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result_plan, table_scan->CreatePlan()); LimitScanResult result; result.splits = result_plan->Splits(); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder.SetReadFieldNames(read_schema) @@ -347,7 +347,7 @@ class DataEvolutionTableTest : public ::testing::Test, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(result.splits)); @@ -361,7 +361,7 @@ class DataEvolutionTableTest : public ::testing::Test, /// the row count derived from them. Reading the splits back is ScanAndReadWithLimit's job, /// which verifies the returned rows after their reader has been destroyed. Result>> PlanSplits(const std::string& table_path) const { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; PAIMON_ASSIGN_OR_RAISE( @@ -369,7 +369,7 @@ class DataEvolutionTableTest : public ::testing::Test, FinishScanContext(scan_context_builder, scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); return plan->Splits(); @@ -439,14 +439,14 @@ class DataEvolutionTableTest : public ::testing::Test, const std::string& table_path, const std::vector>& commit_msgs, std::optional row_id_check_from_snapshot) const { - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_store_commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); file_store_commit->RowIdCheckConflict(row_id_check_from_snapshot); return file_store_commit->Commit(commit_msgs); @@ -505,7 +505,7 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::string anchor, PlannedAnchorFileName(table_path)); ASSERT_OK(CommitDeletionVectors(table_path, base_msgs[0], {{anchor, deleted}})); } - TableLoadOptions scan_builder_load_options{table_path}; + TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder.SetGlobalIndexResult(std::make_shared( @@ -513,7 +513,7 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder, scan_builder_options)); ASSERT_OK_AND_ASSIGN( - auto scan, TableScan::Create(scan_builder_load_options, std::move(scan_context), + auto scan, TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); @@ -545,14 +545,14 @@ class DataEvolutionTableTest : public ::testing::Test, std::shared_ptr bound_predicate, PredicateUtils::CreatePickedFieldFilter(filter_case.predicate, {{"f0", project_row_id ? 3 : 2}})); - TableLoadOptions read_builder_load_options{table_path}; + TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder.SetReadFieldNames(read_fields) .SetPredicate(bound_predicate) .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(read_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); // Reuse the same TableRead with and without scores: the internal row-id @@ -604,7 +604,7 @@ class DataEvolutionTableTest : public ::testing::Test, bool check_scan_plan_when_empty_result = true, bool apply_predicate_to_scan = true) const { // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetPredicate(apply_predicate_to_scan ? predicate : nullptr); @@ -614,7 +614,7 @@ class DataEvolutionTableTest : public ::testing::Test, } PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); @@ -626,13 +626,13 @@ class DataEvolutionTableTest : public ::testing::Test, // read auto splits = result_plan->Splits(); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); @@ -679,7 +679,7 @@ class DataEvolutionTableTest : public ::testing::Test, const std::vector>& expected_first_row_ids, const std::vector& expected_row_counts) { ASSERT_EQ(expected_first_row_ids.size(), expected_row_counts.size()); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetPredicate(predicate); @@ -689,7 +689,7 @@ class DataEvolutionTableTest : public ::testing::Test, } ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1149,23 +1149,23 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -2079,13 +2079,13 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { } auto selected_gap_expected = checked_pointer_cast( arrow::Concatenate(selected_gap_chunks).ValueOrDie()); - TableLoadOptions read_builder_load_options{table_path}; + TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder.SetReadFieldNames({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); const std::vector, std::shared_ptr>> gap_cases = {{gap_split, gap_expected}, @@ -2147,7 +2147,7 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { {std::vector{"id", "embedding", "_ROW_ID", "_SEQUENCE_NUMBER"}, std::vector{"id", "_ROW_ID", "_SEQUENCE_NUMBER"}, std::vector{"_ROW_ID", "_SEQUENCE_NUMBER"}}) { - TableLoadOptions sequence_read_builder_load_options{table_path}; + TableLoadContext sequence_read_builder_load_context{table_path}; std::map sequence_read_builder_options; ReadContextBuilder sequence_read_builder; sequence_read_builder.SetReadFieldNames(projection); @@ -2155,7 +2155,7 @@ TEST_P(DataEvolutionTableTest, TestReadJavaIndependentlyRolledVectorStore) { sequence_read_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr sequence_read, - TableRead::Create(sequence_read_builder_load_options, std::move(sequence_context), + TableRead::Create(sequence_read_builder_load_context, std::move(sequence_context), sequence_read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr sequence_reader, sequence_read->CreateReader(split)); @@ -2264,23 +2264,23 @@ TEST_P(DedicatedVectorTableTest, TestDedicatedVectorStore) { ASSERT_EQ(group[0]->row_count, group[1]->row_count); } - TableLoadOptions partial_builder_load_options{table_path}; + TableLoadContext partial_builder_load_context{table_path}; std::map partial_builder_options; WriteContextBuilder partial_builder("partial"); partial_builder.WithWriteSchema({"embedding"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr partial_context, partial_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(partial_builder_load_options, + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(partial_builder_load_context, std::move(partial_context), partial_builder_options) .status(), "partial writes"); - TableLoadOptions invalid_builder_load_options{table_path}; + TableLoadContext invalid_builder_load_context{table_path}; std::map invalid_builder_options; WriteContextBuilder invalid_builder("invalid"); invalid_builder_options[Options::DATA_EVOLUTION_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_context, invalid_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(invalid_builder_load_options, + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(invalid_builder_load_context, std::move(invalid_context), invalid_builder_options) .status(), "Cannot change immutable option 'data-evolution.enabled' dynamically"); @@ -4048,7 +4048,7 @@ TEST_P(DataEvolutionDvTableTest, TestReadWithDeletionVectorsOnEveryRowRangeGroup // a count query answers from that metadata alone, never reading a row, so the deletion // vectors have to reach it too: without them it reports the 8 rows the files hold - TableLoadOptions count_context_builder_load_options{table_path}; + TableLoadContext count_context_builder_load_context{table_path}; std::map count_context_builder_options; ReadContextBuilder count_context_builder; count_context_builder.SetReadFieldNames(schema->field_names()); @@ -4056,7 +4056,7 @@ TEST_P(DataEvolutionDvTableTest, TestReadWithDeletionVectorsOnEveryRowRangeGroup count_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr count_table_read, - TableRead::Create(count_context_builder_load_options, std::move(count_context), + TableRead::Create(count_context_builder_load_context, std::move(count_context), count_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr count_reader, count_table_read->CreateCountReader(planned_splits)); diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 43cfdcbc4..6175bc8f3 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -26,7 +26,7 @@ #include "fmt/format.h" #include "fmt/ranges.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_factory.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_options.h" @@ -225,16 +225,16 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::vector& write_cols, const std::shared_ptr& write_array) const { // write - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithWriteSchema(write_cols); - write_builder_load_options.file_system = fs_; + write_builder_load_context.file_system = fs_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); ArrowArray c_array; EXPECT_TRUE(arrow::ExportArray(*write_array, &c_array).ok()); @@ -258,16 +258,16 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("commit_user_1"); - commit_builder_load_options.file_system = fs_; + commit_builder_load_context.file_system = fs_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_store_commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -275,14 +275,14 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter Result> ScanData( const std::string& table_path, const std::vector>& partition_filters) const { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetPartitionFilter(partition_filters); - scan_context_builder_load_options.file_system = fs_; + scan_context_builder_load_context.file_system = fs_; PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); @@ -1185,17 +1185,17 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::map& options = {}, const std::shared_ptr& index_result = nullptr, const std::shared_ptr& cache = nullptr) const { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; - scan_context_builder_load_options.file_system = fs_; + scan_context_builder_load_context.file_system = fs_; scan_context_builder.SetPredicate(predicate).SetGlobalIndexResult(index_result); if (cache) { scan_context_builder.WithCache(cache); } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); @@ -1229,16 +1229,16 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::shared_ptr& predicate, const std::shared_ptr& result_plan) const { auto splits = result_plan->Splits(); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; - read_context_builder_load_options.file_system = fs_; + read_context_builder_load_context.file_system = fs_; read_context_builder.SetReadFieldNames(read_schema) .SetPredicate(predicate) .EnableLateMaterializing(false); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); @@ -1624,16 +1624,16 @@ TEST_P(GlobalIndexTest, TestWriteIndexDecodesNestedOrcDictionaries) { ScanData(table_path, /*partition_filters=*/{})); std::shared_ptr indexed_split = std::make_shared(split, std::vector{Range(0, 3)}); - TableLoadOptions read_builder_load_options{table_path}; + TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options; - read_builder_load_options.file_system = fs_; + read_builder_load_context.file_system = fs_; read_builder.SetReadFieldNames({"key", "details", "attributes", "plain_details", "plain_attributes", SpecialFields::RowId().Name()}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(indexed_split)); diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp index 6adf94267..d52809ce4 100644 --- a/test/inte/nested_column_pruning_inte_test.cpp +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -29,7 +29,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/path_util.h" @@ -93,7 +93,7 @@ class NestedColumnPruningInteTest : public ::testing::Test, const std::shared_ptr& expected_schema, const std::string& expected_json, const std::shared_ptr& predicate = nullptr) const { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); @@ -102,7 +102,7 @@ class NestedColumnPruningInteTest : public ::testing::Test, scan_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -110,7 +110,7 @@ class NestedColumnPruningInteTest : public ::testing::Test, auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*expected_schema, c_schema.get()).ok()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder.SetReadSchema(std::move(c_schema)); @@ -118,7 +118,7 @@ class NestedColumnPruningInteTest : public ::testing::Test, read_context_builder.SetPredicate(predicate); } ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -360,13 +360,13 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsAllNonExistent) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions read_context_builder_load_options{table_path_}; + TableLoadContext read_context_builder_load_context{table_path_}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_builder_load_options, + ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -426,13 +426,13 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithNonExistentField) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions read_context_builder_load_options{table_path_}; + TableLoadContext read_context_builder_load_context{table_path_}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_builder_load_options, + ASSERT_NOK_WITH_MSG(TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options), "does not support schema evolution inside struct"); } @@ -509,14 +509,14 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithTypeMismatchShouldFa helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); - TableLoadOptions read_context_builder_load_options{table_path_}; + TableLoadContext read_context_builder_load_context{table_path_}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(data_splits), @@ -604,14 +604,14 @@ TEST_P(NestedColumnPruningInteTest, auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions read_context_builder_load_options{table_path_}; + TableLoadContext read_context_builder_load_context{table_path_}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -794,14 +794,14 @@ TEST_P(NestedColumnPruningInteTest, PruneNestedStructWithSpecialFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions read_context_builder_load_options{table_path_}; + TableLoadContext read_context_builder_load_context{table_path_}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -1211,7 +1211,7 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); - TableLoadOptions write_context_builder_load_options{table_path_}; + TableLoadContext write_context_builder_load_context{table_path_}; std::map write_context_builder_options; WriteContextBuilder write_context_builder("commit_user_1"); write_context_builder_options = options; @@ -1219,7 +1219,7 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { write_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_context_builder_load_context, std::move(write_context), write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { @@ -1240,13 +1240,13 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); - TableLoadOptions commit_context_builder_load_options{table_path_}; + TableLoadContext commit_context_builder_load_context{table_path_}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); @@ -1353,14 +1353,14 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions read_context_builder_load_options{table_path_}; + TableLoadContext read_context_builder_load_context{table_path_}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -1513,14 +1513,14 @@ TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions read_context_builder_load_options{table_path_}; + TableLoadContext read_context_builder_load_context{table_path_}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadSchema(std::move(c_schema)); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); auto create_reader_result = table_read->CreateReader(data_splits); ASSERT_NOK_WITH_MSG(create_reader_result, "partial projection inside list"); diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index 5c4c7e8b9..94cf48339 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -29,7 +29,7 @@ #include "arrow/api.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/variant/generic_variant.h" #include "paimon/data/blob_descriptor.h" #include "paimon/defs.h" @@ -72,7 +72,7 @@ Result> ReadTable( std::string table_path = GetDataDir() + "/" + file_format + "/" + kDatabaseName + ".db/" + table_name; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -81,11 +81,11 @@ Result> ReadTable( scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; @@ -94,7 +94,7 @@ Result> ReadTable( read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index 390f7317b..183a29e5f 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -31,7 +31,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" @@ -352,7 +352,7 @@ class PkCompactionInteTest : public ::testing::Test { int32_t bucket, const std::shared_ptr& write_array, int64_t commit_identifier, bool is_streaming = true, const std::vector& row_kinds = {}, bool write_only = true) const { - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(is_streaming) @@ -363,7 +363,7 @@ class PkCompactionInteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); @@ -377,14 +377,14 @@ class PkCompactionInteTest : public ::testing::Test { Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("commit_user_1"); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_store_commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); return file_store_commit->Commit(commit_msgs); } @@ -406,7 +406,7 @@ class PkCompactionInteTest : public ::testing::Test { const std::string& table_path, const std::map& partition, int32_t bucket, bool full_compaction, int64_t commit_identifier, const std::map& options = {}) { - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder_options = options; @@ -416,7 +416,7 @@ class PkCompactionInteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); PAIMON_RETURN_NOT_OK(file_store_write->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( @@ -436,7 +436,7 @@ class PkCompactionInteTest : public ::testing::Test { std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(table_path, options, /*is_streaming_mode=*/false)); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -449,7 +449,7 @@ class PkCompactionInteTest : public ::testing::Test { scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); std::vector> data_splits = result_plan->Splits(); @@ -475,7 +475,7 @@ class PkCompactionInteTest : public ::testing::Test { auto iter = expected_data_per_partition_bucket.find(key); ASSERT_TRUE(iter != expected_data_per_partition_bucket.end()) << "Unexpected partition=" << key.first << " bucket=" << key.second; - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; @@ -487,7 +487,7 @@ class PkCompactionInteTest : public ::testing::Test { read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); @@ -514,7 +514,7 @@ class PkCompactionInteTest : public ::testing::Test { template void ScanAllRows(const std::string& table_path, Fn consume) { std::map options = {{Options::FILE_SYSTEM, "local"}}; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -524,10 +524,10 @@ class PkCompactionInteTest : public ::testing::Test { scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; @@ -536,7 +536,7 @@ class PkCompactionInteTest : public ::testing::Test { read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -651,7 +651,7 @@ TEST_F(PkCompactionInteTest, TestMetadataOnlyLevelUpgradeKeepsValueStats) { ASSERT_FALSE(before->value_stats == SimpleStats::EmptyStats()); ASSERT_EQ(before->value_stats, after->value_stats); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::FILE_SYSTEM] = "local"; @@ -659,7 +659,7 @@ TEST_F(PkCompactionInteTest, TestMetadataOnlyLevelUpgradeKeepsValueStats) { scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1u, plan->Splits().size()); @@ -1034,12 +1034,12 @@ TEST_F(PkCompactionInteTest, DeduplicateWithBitmap64HighPositions) { ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)); // Reload committed metadata and DV bytes; no test-created DV or lookup result is injected. - TableLoadOptions scan_builder_load_options{table_path}; + TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); int32_t dv_count = 0; @@ -1787,14 +1787,14 @@ TEST_F(PkCompactionInteTest, CompactWithSchemaEvolution) { // Step 5: Read and verify using DataSplits from CompactAfter in write CommitMessages. { - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = "parquet"; ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); // Use the latest snapshot id for both splits. @@ -2057,7 +2057,7 @@ TEST_F(PkCompactionInteTest, ContinuousWriteWithBackgroundCompact) { // Step 1-4: Streaming writes with background compact in one FileStoreWrite. { - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory( @@ -2065,7 +2065,7 @@ TEST_F(PkCompactionInteTest, ContinuousWriteWithBackgroundCompact) { ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); auto write_batch = [&](const std::string& json_data) -> Status { @@ -2270,14 +2270,14 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { auto data_type = arrow::struct_(fields); // Step 2: Write data to the branch-rt and compact. - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder_options["branch"] = "rt"; write_builder.WithStreamingMode(true).WithTempDirectory(PathUtil::JoinPath(dir_->Str(), "tmp")); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); // dt=20240726, bucket-0 only has ["20240726", "cherry", 3] one row @@ -2329,7 +2329,7 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(compact_after[0]->row_count, 2); // Step 4: Commit to branch-rt. - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("commit_user_1"); commit_builder_options[Options::FILE_SYSTEM] = "local"; @@ -2338,7 +2338,7 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr file_store_commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); @@ -2358,7 +2358,7 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { // Step 5: Scan and read branch-rt. std::map branch_options = {{Options::FILE_SYSTEM, "local"}, {Options::BRANCH, "rt"}}; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = branch_options; @@ -2366,7 +2366,7 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); @@ -2378,14 +2378,14 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch) { ASSERT_EQ(split_impl->SnapshotId(), 3); } - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = branch_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp index 88f9fb24b..3dc109de3 100644 --- a/test/inte/primary_key_sorted_index_inte_test.cpp +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -31,7 +31,7 @@ #include "arrow/api.h" #include "fmt/format.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/table/source/data_split_impl.h" @@ -200,7 +200,7 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::optional& snapshot_id, bool index_enabled, const std::vector>& partition_filters, const std::string& branch, const std::shared_ptr& file_system) const { - TableLoadOptions builder_load_options{TablePath(partitioned)}; + TableLoadContext builder_load_context{TablePath(partitioned)}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::GLOBAL_INDEX_ENABLED] = index_enabled ? "true" : "false"; @@ -215,12 +215,12 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, builder_options[Options::BRANCH] = branch; } if (file_system != nullptr) { - builder_load_options.file_system = file_system; + builder_load_context.file_system = file_system; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_scan, - TableScan::Create(builder_load_options, std::move(context), builder_options)); + TableScan::Create(builder_load_context, std::move(context), builder_options)); return table_scan->CreatePlan(); } @@ -228,7 +228,7 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, const std::vector>& splits, const std::map& options, const std::shared_ptr& file_system) const { - TableLoadOptions builder_load_options{TablePath(partitioned)}; + TableLoadContext builder_load_context{TablePath(partitioned)}; std::map builder_options; ReadContextBuilder builder; if (partitioned) { @@ -241,12 +241,12 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, builder_options[key] = value; } if (file_system != nullptr) { - builder_load_options.file_system = file_system; + builder_load_context.file_system = file_system; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_read, - TableRead::Create(builder_load_options, std::move(context), builder_options)); + TableRead::Create(builder_load_context, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -548,7 +548,7 @@ TEST_P(PrimaryKeySortedIndexInteTest, ScoredSplitFailsBeforeForceKeepDeleteFallb inner_split, std::vector{Range(first_range.from, first_range.from)}, std::vector{0.5F}); - TableLoadOptions builder_load_options{TablePath(/*partitioned=*/false)}; + TableLoadContext builder_load_context{TablePath(/*partitioned=*/false)}; std::map builder_options; ReadContextBuilder builder; builder.SetReadFieldNames({"id", "score", "tag"}) @@ -557,7 +557,7 @@ TEST_P(PrimaryKeySortedIndexInteTest, ScoredSplitFailsBeforeForceKeepDeleteFallb ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_read, - TableRead::Create(builder_load_options, std::move(context), builder_options)); + TableRead::Create(builder_load_context, std::move(context), builder_options)); auto* key_value_table_read = dynamic_cast(table_read.get()); ASSERT_TRUE(key_value_table_read != nullptr); key_value_table_read->ForceKeepDelete(true); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index af1ef60e2..53da9fd35 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -41,7 +41,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" @@ -284,7 +284,7 @@ Result> ReadSystemTable( const std::string& system_table_path, const std::map& options, bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, const std::vector& read_field_names = {}, bool read_next_plan = false) { - TableLoadOptions scan_context_builder_load_options{system_table_path}; + TableLoadContext scan_context_builder_load_context{system_table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -296,14 +296,14 @@ Result> ReadSystemTable( scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); if (read_next_plan) { PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan()); } - TableLoadOptions read_context_builder_load_options{system_table_path}; + TableLoadContext read_context_builder_load_context{system_table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; @@ -318,7 +318,7 @@ Result> ReadSystemTable( read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); @@ -391,7 +391,7 @@ TEST_P(ReadInteTest, TestAppendSimple) { DataField(2, arrow::field("f2", arrow::int32())), DataField(3, arrow::field("f3", arrow::float64()))}; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = param.file_format; @@ -402,11 +402,11 @@ TEST_P(ReadInteTest, TestAppendSimple) { context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); if (specific_table_schema) { - context_builder_load_options.table_schema_json = specific_table_schema.value(); + context_builder_load_context.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list; @@ -479,7 +479,7 @@ TEST_P(ReadInteTest, TestReadWithLimits) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = param.file_format; @@ -495,7 +495,7 @@ TEST_P(ReadInteTest, TestReadWithLimits) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list; @@ -548,7 +548,7 @@ TEST_P(ReadInteTest, TestReadAheadCacheMetrics) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = param.file_format; @@ -559,7 +559,7 @@ TEST_P(ReadInteTest, TestReadAheadCacheMetrics) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list; @@ -627,7 +627,7 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) { DataField(0, arrow::field("dt", arrow::utf8())), }; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = param.file_format; @@ -640,7 +640,7 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) { context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector> data_splits; @@ -691,26 +691,26 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ArrowSchemaRelease(&schema); std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); - TableLoadOptions scan_context_builder_load_options{system_table_path}; + TableLoadContext scan_context_builder_load_context{system_table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); - TableLoadOptions read_context_builder_load_options{system_table_path}; + TableLoadContext read_context_builder_load_context{system_table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -733,26 +733,26 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; - TableLoadOptions scan_context_builder_load_options{system_table_path}; + TableLoadContext scan_context_builder_load_context{system_table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); - TableLoadOptions read_context_builder_load_options{system_table_path}; + TableLoadContext read_context_builder_load_context{system_table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1169,7 +1169,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::shared_ptr predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); - TableLoadOptions ro_scan_context_builder_load_options{table_path + "$ro"}; + TableLoadContext ro_scan_context_builder_load_context{table_path + "$ro"}; std::map ro_scan_context_builder_options; ScanContextBuilder ro_scan_context_builder; ro_scan_context_builder_options = options; @@ -1178,13 +1178,13 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP ro_scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr ro_table_scan, - TableScan::Create(ro_scan_context_builder_load_options, std::move(ro_scan_context), + TableScan::Create(ro_scan_context_builder_load_context, std::move(ro_scan_context), ro_scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); ASSERT_EQ(ro_file_count, 1); - TableLoadOptions read_context_builder_load_options{table_path + "$ro"}; + TableLoadContext read_context_builder_load_context{table_path + "$ro"}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; // Do not enable row-level predicate filtering: the result must come from the file reader's @@ -1193,7 +1193,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP read_context_builder.SetPredicate(predicate).SetReadFieldNames({"k", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(ro_plan->Splits())); @@ -1238,14 +1238,14 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); - TableLoadOptions scan_context_builder_load_options{table_path + "$ro"}; + TableLoadContext scan_context_builder_load_context{table_path + "$ro"}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); @@ -1255,14 +1255,14 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { }); auto c_projected_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_projected_schema.get()).ok()); - TableLoadOptions read_context_builder_load_options{table_path + "$ro"}; + TableLoadContext read_context_builder_load_context{table_path + "$ro"}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadSchema(std::move(c_projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); @@ -1494,7 +1494,7 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite /*is_streaming_mode=*/true)); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); - TableLoadOptions write_context_builder_load_options{table_path}; + TableLoadContext write_context_builder_load_context{table_path}; std::map write_context_builder_options; WriteContextBuilder write_context_builder("partial-write"); write_context_builder.WithWriteSchema({"score"}); @@ -1502,7 +1502,7 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite write_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr write, - FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_context_builder_load_context, std::move(write_context), write_context_builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr batch, @@ -1513,14 +1513,14 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite write->PrepareCommit()); ASSERT_OK(write->Close()); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("partial-write"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_messages)); @@ -1887,7 +1887,7 @@ TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadO for (bool direct : {false, true}) { SCOPED_TRACE(direct); const std::string path = table->GetRootPath() + "$" + kind; - TableLoadOptions scan_context_builder_48980_load_options{path}; + TableLoadContext scan_context_builder_48980_load_context{path}; std::map scan_context_builder_48980_options; ScanContextBuilder scan_context_builder_48980; scan_context_builder_48980_options = read_options; @@ -1900,11 +1900,11 @@ TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadO table->Copy(scan_context_builder_48980_options)); return configured->NewScan(std::move(scan_context)); })() - : TableScan::Create(scan_context_builder_48980_load_options, + : TableScan::Create(scan_context_builder_48980_load_context, std::move(scan_context), scan_context_builder_48980_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - TableLoadOptions read_context_builder_93839_load_options{path}; + TableLoadContext read_context_builder_93839_load_context{path}; std::map read_context_builder_93839_options; ReadContextBuilder read_context_builder_93839; read_context_builder_93839_options = read_options; @@ -1917,7 +1917,7 @@ TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadO table->Copy(read_context_builder_93839_options)); return configured->NewRead(std::move(read_context)); })() - : TableRead::Create(read_context_builder_93839_load_options, + : TableRead::Create(read_context_builder_93839_load_context, std::move(read_context), read_context_builder_93839_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, @@ -1967,7 +1967,7 @@ TEST(SystemTableReadInteTest, TestCopiedChangelogHonorsDynamicSequenceNumber) { for (bool direct : {false, true}) { SCOPED_TRACE(direct); const std::string path = table->GetRootPath() + "$" + kind; - TableLoadOptions scan_load{path}; + TableLoadContext scan_load{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN( @@ -1975,7 +1975,7 @@ TEST(SystemTableReadInteTest, TestCopiedChangelogHonorsDynamicSequenceNumber) { direct ? configured->NewScan(std::move(scan_context)) : TableScan::Create(scan_load, std::move(scan_context), read_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - TableLoadOptions read_load{path}; + TableLoadContext read_load{path}; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN( @@ -2288,7 +2288,7 @@ TEST_P(ReadInteTest, TestAppendReadWithMultipleBuckets) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); @@ -2303,7 +2303,7 @@ TEST_P(ReadInteTest, TestAppendReadWithMultipleBuckets) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -2375,7 +2375,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f3", "f0", "f1"}); @@ -2389,7 +2389,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicate) { param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -2480,7 +2480,7 @@ TEST_P(ReadInteTest, TestAppendReadWithComplexTypePredicate) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_data.db/append_complex_data"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -2495,7 +2495,7 @@ TEST_P(ReadInteTest, TestAppendReadWithComplexTypePredicate) { context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -2559,7 +2559,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -2576,7 +2576,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -2649,7 +2649,7 @@ TEST_P(ReadInteTest, TestAppendReadWithLateMaterializing) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -2665,7 +2665,7 @@ TEST_P(ReadInteTest, TestAppendReadWithLateMaterializing) { .EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -2733,7 +2733,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -2750,7 +2750,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -2825,8 +2825,8 @@ TEST_P(ReadInteTest, TestAppendReadIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - TableLoadOptions context_builder_load_options; - context_builder_load_options.path = + TableLoadContext context_builder_load_context; + context_builder_load_context.path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09/"; std::map context_builder_options; ReadContextBuilder context_builder; @@ -2840,7 +2840,7 @@ TEST_P(ReadInteTest, TestAppendReadIOException) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); Result> table_read = TableRead::Create( - context_builder_load_options, std::move(read_context), context_builder_options); + context_builder_load_context, std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(data_splits); @@ -2875,7 +2875,7 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVectorSimple) { data_file = "data-ed5d7184-cf42-4f4a-bf83-1e3080d9012d-0.parquet"; dv_file = "index/index-62372b7b-d1cf-4dde-a162-c9d9d35006b4-1"; } - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = param.file_format; @@ -2888,7 +2888,7 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVectorSimple) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); DataSplitsDv input_data_splits = { @@ -2929,7 +2929,7 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVector) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -2941,7 +2941,7 @@ TEST_P(ReadInteTest, TestPkTableWithDeletionVector) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list; @@ -3000,7 +3000,7 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot6) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(15.0)); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3016,7 +3016,7 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot6) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3091,7 +3091,7 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot8) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3105,7 +3105,7 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot8) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3175,7 +3175,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolution) { DataField(5, arrow::field("a", arrow::int32())), DataField(8, arrow::field("e", arrow::int32()))}; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3186,13 +3186,13 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolution) { param.enable_adaptive_prefetch_strategy; context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder_load_options.table_schema_json = specific_table_schema.value(); + context_builder_load_context.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3287,7 +3287,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithReaderBuildParallelism /*schema ids*/ {1, 0}}}; auto data_splits = CreateDataSplits(input_data_splits, /*snapshot_id=*/2); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3299,7 +3299,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithReaderBuildParallelism context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -3357,7 +3357,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateFilter) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/append_table_with_alter_table/"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3374,7 +3374,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateFilter) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3442,7 +3442,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateOnlyPushDown) std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_with_alter_table.db/" "append_table_with_alter_table/"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3457,7 +3457,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateOnlyPushDown) ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3521,7 +3521,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot5WithSchemaEvolution) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3535,7 +3535,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot5WithSchemaEvolution) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3612,7 +3612,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolution) { auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3626,7 +3626,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolution) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3703,7 +3703,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); @@ -3717,7 +3717,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush context_builder.EnablePrefetch(param.enable_prefetch).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3793,7 +3793,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithLateMaterializing FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); @@ -3807,7 +3807,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithLateMaterializing context_builder.EnableLateMaterializing(true).EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3881,7 +3881,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter) FieldType::INT, Literal(510)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3897,7 +3897,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter) ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -3979,7 +3979,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithBuildInFieldId) { "data-11b12094-192f-4ad8-92a8-ae8cba5e25ef-0.parquet"}; } - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -3993,7 +3993,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithBuildInFieldId) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); DataSplitsSchema input_data_splits = { @@ -4048,7 +4048,7 @@ TEST_P(ReadInteTest, TestAppendReadNestedType) { DataField(7, arrow::field("f6", arrow::decimal128(2, 2)))}; std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -4061,7 +4061,7 @@ TEST_P(ReadInteTest, TestAppendReadNestedType) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list; @@ -4109,7 +4109,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCast) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -4123,7 +4123,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCast) { ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -4197,7 +4197,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCastWithPredicatePushD std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); @@ -4212,7 +4212,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCastWithPredicatePushD ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list_0; @@ -4282,7 +4282,7 @@ TEST_P(ReadInteTest, TestReadWithPKFallBackBranch) { DataField(2, arrow::field("amount", arrow::int32())), }; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -4291,11 +4291,11 @@ TEST_P(ReadInteTest, TestReadWithPKFallBackBranch) { param.enable_adaptive_prefetch_strategy; context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder_load_options.table_schema_json = specific_table_schema.value(); + context_builder_load_context.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -4342,14 +4342,14 @@ TEST_P(ReadInteTest, TestReadWithAppendFallBackBranch) { data_splits.push_back(split); } - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.EnablePrefetch(param.enable_prefetch); context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -4390,7 +4390,7 @@ TEST_P(ReadInteTest, TestFallBackBranchStreamRead) { std::vector read_fields = {DataField(0, arrow::field("dt", arrow::utf8())), DataField(1, arrow::field("name", arrow::utf8())), DataField(2, arrow::field("amount", arrow::int32()))}; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -4399,7 +4399,7 @@ TEST_P(ReadInteTest, TestFallBackBranchStreamRead) { context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_split)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -4440,7 +4440,7 @@ TEST_P(ReadInteTest, TestReadWithPKRtBranch) { DataField(2, arrow::field("amount", arrow::int32())), }; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -4449,11 +4449,11 @@ TEST_P(ReadInteTest, TestReadWithPKRtBranch) { context_builder_options["branch"] = "rt"; context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder_load_options.table_schema_json = specific_table_schema.value(); + context_builder_load_context.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -4501,7 +4501,7 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) { DataField(2, arrow::field("value2", arrow::int32())), }; - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -4510,11 +4510,11 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) { context_builder_options["branch"] = "test"; context_builder.EnablePrefetch(param.enable_prefetch); if (specific_table_schema) { - context_builder_load_options.table_schema_json = specific_table_schema.value(); + context_builder_load_context.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); ASSERT_OK_AND_ASSIGN(auto result_array, @@ -4636,7 +4636,7 @@ TEST_P(ReadInteTest, TestSpecificFs) { auto countable_fs = std::make_shared(std::make_shared(), &io_count); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder.SetReadAheadCacheEnabled(param.read_ahead_cache_enabled); @@ -4644,11 +4644,11 @@ TEST_P(ReadInteTest, TestSpecificFs) { context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; context_builder_options["orc.read.enable-metrics"] = "true"; - context_builder_load_options.file_system = countable_fs; + context_builder_load_context.file_system = countable_fs; context_builder.EnablePrefetch(param.enable_prefetch); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); std::vector file_list; diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 640c03487..7551ba324 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -45,7 +45,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/predicate/equal.h" @@ -545,7 +545,7 @@ class RealtimeWriteInteTest : public ::testing::Test { Result> CreateRealtimeWriter( const std::shared_ptr& realtime_context, const std::string& temp_directory) const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; WriteContextBuilder builder(commit_user_); builder_options = options_; @@ -554,7 +554,7 @@ class RealtimeWriteInteTest : public ::testing::Test { builder.WithTempDirectory(temp_directory); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return FileStoreWrite::Create(builder_load_options, std::move(context), builder_options); + return FileStoreWrite::Create(builder_load_context, std::move(context), builder_options); } Result> CreateRealtimeWriter() const { @@ -728,14 +728,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result Commit(const std::vector& realtime_commits, int64_t commit_identifier) const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder(commit_user_); builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); return commit->CommitWithProgress(realtime_commits, commit_identifier, /*watermark=*/std::nullopt); } @@ -756,14 +756,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result DropPartition(const std::map& partition, int64_t commit_identifier) const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder(kDropPartitionCommitUser); builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->DropPartition({partition}, commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -776,14 +776,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result TruncateTable(int64_t commit_identifier) const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder(kTruncateCommitUser); builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->TruncateTable(commit_identifier)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_)); SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_); @@ -796,14 +796,14 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result RollbackToAsLatest(int64_t target_snapshot_id) const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder(kRollbackCommitUser); builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(bool rolled_back, commit->RollbackToAsLatest(target_snapshot_id)); if (!rolled_back) { return Status::Invalid("failed to commit rollback snapshot"); @@ -821,7 +821,7 @@ class RealtimeWriteInteTest : public ::testing::Test { Result>> CompactAndPrepareCommit( const std::vector& partition_buckets, int64_t commit_identifier, bool full_compaction, std::unique_ptr* compactor) const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; WriteContextBuilder builder("regression-compactor"); builder_options = options_; @@ -829,7 +829,7 @@ class RealtimeWriteInteTest : public ::testing::Test { builder.WithStreamingMode(true).Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr writer, - FileStoreWrite::Create(builder_load_options, std::move(context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(context), builder_options)); for (const RealtimePartitionBucket& scope : partition_buckets) { PAIMON_RETURN_NOT_OK(writer->Compact(scope.partition, scope.bucket, full_compaction)); } @@ -841,14 +841,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CommitCompaction(const std::vector>& messages, int64_t commit_identifier) const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder("regression-compactor"); builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); PAIMON_RETURN_NOT_OK(commit->Commit(messages, commit_identifier)); SnapshotManager snapshot_manager(dir_->GetFileSystem(), table_path_); PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, snapshot_manager.LatestSnapshot()); @@ -861,7 +861,7 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CompactAndCommit(const std::map& partition, int32_t bucket, int64_t commit_identifier, bool full_compaction = true) const { - TableLoadOptions write_builder_load_options{table_path_}; + TableLoadContext write_builder_load_context{table_path_}; std::map write_builder_options; WriteContextBuilder write_builder(commit_user_); write_builder_options = options_; @@ -870,7 +870,7 @@ class RealtimeWriteInteTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr compaction_writer, - FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); PAIMON_RETURN_NOT_OK(compaction_writer->Compact(partition, bucket, full_compaction)); PAIMON_ASSIGN_OR_RAISE( @@ -880,7 +880,7 @@ class RealtimeWriteInteTest : public ::testing::Test { return Status::Invalid("compaction did not produce a commit message"); } - TableLoadOptions commit_builder_load_options{table_path_}; + TableLoadContext commit_builder_load_context{table_path_}; std::map commit_builder_options; CommitContextBuilder commit_builder(commit_user_); commit_builder_options = options_; @@ -888,7 +888,7 @@ class RealtimeWriteInteTest : public ::testing::Test { commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr compaction_commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); PAIMON_RETURN_NOT_OK(compaction_commit->Commit(compaction_messages, commit_identifier)); PAIMON_RETURN_NOT_OK(compaction_writer->Close()); @@ -914,21 +914,21 @@ class RealtimeWriteInteTest : public ::testing::Test { } Result ExpireSnapshots() const { - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder(commit_user_); builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); return commit->Expire(); } Result> CreatePlan( const std::shared_ptr& realtime_context, const std::shared_ptr& predicate) const { - TableLoadOptions scan_builder_load_options{table_path_}; + TableLoadContext scan_builder_load_context{table_path_}; std::map scan_builder_options; ScanContextBuilder scan_builder; if (realtime_context) { @@ -938,7 +938,7 @@ class RealtimeWriteInteTest : public ::testing::Test { scan_builder_options = options_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); return scan->CreatePlan(); } @@ -946,7 +946,7 @@ class RealtimeWriteInteTest : public ::testing::Test { Result> CreateQueryReader( const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; @@ -955,7 +955,7 @@ class RealtimeWriteInteTest : public ::testing::Test { .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); return table_read->CreateReader(plan->Splits()); } @@ -971,7 +971,7 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::shared_ptr& plan, const std::shared_ptr& realtime_context, const std::vector& read_fields, const std::shared_ptr& predicate, bool enable_predicate_filter) const { - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; @@ -982,7 +982,7 @@ class RealtimeWriteInteTest : public ::testing::Test { .WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); @@ -997,7 +997,7 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::string& expected_json) const { std::unique_ptr c_read_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok()); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; @@ -1006,7 +1006,7 @@ class RealtimeWriteInteTest : public ::testing::Test { .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); @@ -1093,14 +1093,14 @@ class RealtimeWriteInteTest : public ::testing::Test { Result CountRows(const std::shared_ptr& plan, const std::shared_ptr& realtime_context) const { - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; read_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr count_reader, table_read->CreateCountReader(plan->Splits())); @@ -1394,44 +1394,44 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - TableLoadOptions write_builder_load_options{table_path_}; + TableLoadContext write_builder_load_context{table_path_}; std::map write_builder_options; WriteContextBuilder write_builder(commit_user_); write_builder_options = disabled_options; write_builder.WithStreamingMode(true).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(write_builder_load_options, std::move(write_context), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options), "real-time write requires realtime.enabled=true"); - TableLoadOptions scan_builder_load_options{table_path_}; + TableLoadContext scan_builder_load_context{table_path_}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options = disabled_options; scan_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options), "real-time scan requires realtime.enabled=true"); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = disabled_options; read_builder.WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_NOK_WITH_MSG( - TableRead::Create(read_builder_load_options, std::move(read_context), read_builder_options), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options), "real-time read requires realtime.enabled=true"); - TableLoadOptions commit_builder_load_options{table_path_}; + TableLoadContext commit_builder_load_context{table_path_}; std::map commit_builder_options; CommitContextBuilder commit_builder(commit_user_); commit_builder_options = disabled_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, @@ -1532,16 +1532,16 @@ TEST_F(RealtimeWriteInteTest, TestRefreshPreparationAllowsScanWriteAndStoreRegis ConfigureRealtimeDeduplicateTable(); auto fs = std::make_shared(dir_->GetFileSystem()); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; WriteContextBuilder builder(commit_user_); builder_options = options_; - builder_load_options.file_system = fs; + builder_load_context.file_system = fs; builder.WithStreamingMode(true).WithRealtimeContext(context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr writer, - FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); const std::vector initial = {{1, "old", "p0"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(initial, /*partitioned=*/false)); @@ -1645,16 +1645,16 @@ TEST_F(RealtimeWriteInteTest, TestRefreshPreparationFailureKeepsProgressAndCanRe ConfigureRealtimeDeduplicateTable(); auto fs = std::make_shared(dir_->GetFileSystem()); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, RealtimeContext::Create()); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; WriteContextBuilder builder(commit_user_); builder_options = options_; - builder_load_options.file_system = fs; + builder_load_context.file_system = fs; builder.WithStreamingMode(true).WithRealtimeContext(context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr writer, - FileStoreWrite::Create(builder_load_options, std::move(write_context), builder_options)); + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); const std::vector initial = {{1, "old", "p0"}}; ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(initial, /*partitioned=*/false)); @@ -1905,7 +1905,7 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) auto concurrent_message = std::make_shared( initial_message->Partition(), /*bucket=*/0, /*total_buckets=*/1, DataIncrement({}, {}, {}, {concurrent_dv}, {}), CompactIncrement({}, {}, {}, {}, {})); - TableLoadOptions concurrent_builder_load_options{table_path_}; + TableLoadContext concurrent_builder_load_context{table_path_}; std::map concurrent_builder_options; CommitContextBuilder concurrent_builder("concurrent-dv-writer"); concurrent_builder_options = options_; @@ -1913,7 +1913,7 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) concurrent_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr concurrent_commit, - FileStoreCommit::Create(concurrent_builder_load_options, std::move(concurrent_context), + FileStoreCommit::Create(concurrent_builder_load_context, std::move(concurrent_context), concurrent_builder_options)); bool interleaved = false; auto fs = std::make_shared( @@ -1922,15 +1922,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) return concurrent_commit->Commit({concurrent_message}, /*commit_identifier=*/0, /*watermark=*/std::nullopt); }); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder(commit_user_); builder_options = options_; - builder_load_options.file_system = fs; + builder_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/1, /*watermark=*/std::nullopt)); @@ -1992,15 +1992,15 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryRemapsCompactedO interleaved = true; return Status::OK(); }); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder(commit_user_); builder_options = options_; - builder_load_options.file_system = fs; + builder_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot_id, commit->CommitWithProgress(progress, /*commit_identifier=*/2, /*watermark=*/std::nullopt)); @@ -2047,14 +2047,14 @@ TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompa ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot)); } - TableLoadOptions write_builder_load_options{table_path_}; + TableLoadContext write_builder_load_context{table_path_}; std::map write_builder_options; WriteContextBuilder write_builder("compactor"); write_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr compactor, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr forbidden_batch, MakeBatch({{99, "not-written", "p0"}}, /*partitioned=*/false)); @@ -2128,15 +2128,15 @@ TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompa } else { ASSERT_OK(publish_build()); } - TableLoadOptions commit_builder_load_options{table_path_}; + TableLoadContext commit_builder_load_context{table_path_}; std::map commit_builder_options; CommitContextBuilder commit_builder("compactor"); commit_builder_options = options_; - commit_builder_load_options.file_system = fs; + commit_builder_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); ASSERT_OK(commit->Commit(messages, /*commit_identifier=*/0)); ASSERT_TRUE(interleaved); @@ -2263,14 +2263,14 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCompactionRejectsReplacedIn ASSERT_EQ(1, initial_snapshot); ASSERT_OK(writer->Close()); - TableLoadOptions write_builder_load_options{table_path_}; + TableLoadContext write_builder_load_context{table_path_}; std::map write_builder_options; WriteContextBuilder write_builder("stale-compactor"); write_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr compactor, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); ASSERT_OK(compactor->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); ASSERT_OK_AND_ASSIGN( @@ -2284,14 +2284,14 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCompactionRejectsReplacedIn ASSERT_OK_AND_ASSIGN(Snapshot replacement, CompactAndCommit(/*partition=*/{}, /*bucket=*/0, /*commit_identifier=*/1)); ASSERT_EQ(2, replacement.Id()); - TableLoadOptions commit_builder_load_options{table_path_}; + TableLoadContext commit_builder_load_context{table_path_}; std::map commit_builder_options; CommitContextBuilder commit_builder("stale-compactor"); commit_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); ASSERT_NOK_WITH_MSG(commit->Commit(messages, /*commit_identifier=*/0), "input is no longer active"); @@ -2834,14 +2834,14 @@ TEST_F(RealtimeWriteInteTest, TestNoHitPreservesBaselineDv) { auto no_hit = std::make_shared(); no_hit->Add(0); progress[0].deleted_offsets = no_hit; - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder(commit_user_); builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN( int64_t snapshot, commit->CommitWithProgress(progress, /*commit_identifier=*/4, /*watermark=*/std::nullopt)); @@ -2965,15 +2965,15 @@ TEST_F(RealtimeWriteInteTest, TestPublicationConflictCleansOnlyAttemptDv) { } return Status::OK(); }); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder("regression-compactor"); builder_options = options_; - builder_load_options.file_system = fs; + builder_load_context.file_system = fs; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK(commit->Commit(messages, /*commit_identifier=*/0)); ASSERT_OK_AND_ASSIGN(uint64_t attempts, commit->GetCommitMetrics()->GetCounter( CommitMetrics::LAST_COMMIT_ATTEMPTS)); @@ -3906,25 +3906,25 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesSnapshotConflict) { const std::string first_snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"); auto conflict_file_system = std::make_shared(first_snapshot_path); - TableLoadOptions bucket0_builder_load_options{table_path_}; + TableLoadContext bucket0_builder_load_context{table_path_}; std::map bucket0_builder_options; CommitContextBuilder bucket0_builder("bucket0_realtime_commit_user"); bucket0_builder_options = options_; - bucket0_builder_load_options.file_system = conflict_file_system; + bucket0_builder_load_context.file_system = conflict_file_system; ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket0_context, bucket0_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr bucket0_commit, - FileStoreCommit::Create(bucket0_builder_load_options, std::move(bucket0_context), + FileStoreCommit::Create(bucket0_builder_load_context, std::move(bucket0_context), bucket0_builder_options)); - TableLoadOptions bucket1_builder_load_options{table_path_}; + TableLoadContext bucket1_builder_load_context{table_path_}; std::map bucket1_builder_options; CommitContextBuilder bucket1_builder("bucket1_realtime_commit_user"); bucket1_builder_options = options_; - bucket1_builder_load_options.file_system = conflict_file_system; + bucket1_builder_load_context.file_system = conflict_file_system; ASSERT_OK_AND_ASSIGN(std::unique_ptr bucket1_context, bucket1_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr bucket1_commit, - FileStoreCommit::Create(bucket1_builder_load_options, std::move(bucket1_context), + FileStoreCommit::Create(bucket1_builder_load_context, std::move(bucket1_context), bucket1_builder_options)); std::optional> bucket0_result; @@ -3993,7 +3993,7 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id)); } - TableLoadOptions compact_write_builder_load_options{table_path_}; + TableLoadContext compact_write_builder_load_context{table_path_}; std::map compact_write_builder_options; WriteContextBuilder compact_write_builder("compaction_write_user"); compact_write_builder_options = options_; @@ -4003,7 +4003,7 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { compact_write_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr compact_writer, - FileStoreWrite::Create(compact_write_builder_load_options, std::move(compact_write_context), + FileStoreWrite::Create(compact_write_builder_load_context, std::move(compact_write_context), compact_write_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); @@ -4028,27 +4028,27 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) { PathUtil::JoinPath(table_path_, "snapshot/snapshot-3"); auto conflict_file_system = std::make_shared(conflict_snapshot_path); - TableLoadOptions append_commit_builder_load_options{table_path_}; + TableLoadContext append_commit_builder_load_context{table_path_}; std::map append_commit_builder_options; CommitContextBuilder append_commit_builder(commit_user_); append_commit_builder_options = options_; - append_commit_builder_load_options.file_system = conflict_file_system; + append_commit_builder_load_context.file_system = conflict_file_system; ASSERT_OK_AND_ASSIGN(std::unique_ptr append_commit_context, append_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr append_commit, - FileStoreCommit::Create(append_commit_builder_load_options, + FileStoreCommit::Create(append_commit_builder_load_context, std::move(append_commit_context), append_commit_builder_options)); - TableLoadOptions compact_commit_builder_load_options{table_path_}; + TableLoadContext compact_commit_builder_load_context{table_path_}; std::map compact_commit_builder_options; CommitContextBuilder compact_commit_builder("compaction_commit_user"); compact_commit_builder_options = options_; - compact_commit_builder_load_options.file_system = conflict_file_system; + compact_commit_builder_load_context.file_system = conflict_file_system; ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_commit_context, compact_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr compact_commit, - FileStoreCommit::Create(compact_commit_builder_load_options, + FileStoreCommit::Create(compact_commit_builder_load_context, std::move(compact_commit_context), compact_commit_builder_options)); std::optional> append_result; @@ -4313,7 +4313,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) { ASSERT_OK_AND_ASSIGN(std::shared_ptr lifetime_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; @@ -4322,7 +4322,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRead) { .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(lifetime_plan->Splits())); @@ -4391,14 +4391,14 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::vector disk_rows, ReadRows()); ASSERT_EQ(rows, disk_rows); - TableLoadOptions scan_builder_load_options{table_path_ + "$ro"}; + TableLoadContext scan_builder_load_context{table_path_ + "$ro"}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options = options_; scan_builder.WithRealtimeContext(realtime_context).WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); Result> scan = - TableScan::Create(scan_builder_load_options, std::move(scan_context), scan_builder_options); + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options); ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString(); ASSERT_NE(std::string::npos, scan.status().ToString().find( "PK real-time union read does not support read-optimized")); @@ -4428,7 +4428,7 @@ TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) { CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_OK_AND_ASSIGN(std::shared_ptr reader_plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; @@ -4437,7 +4437,7 @@ TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) { .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_reader, table_read->CreateReader(reader_plan->Splits())); @@ -4933,7 +4933,7 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) { }); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; @@ -4942,7 +4942,7 @@ TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) { .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); @@ -5231,14 +5231,14 @@ TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) { TEST_F(RealtimeWriteInteTest, TestPkRecovery) { CreatePkTable(); - TableLoadOptions seed_builder_load_options{table_path_}; + TableLoadContext seed_builder_load_context{table_path_}; std::map seed_builder_options; WriteContextBuilder seed_builder(commit_user_); seed_builder_options = options_; seed_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_context, seed_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_writer, - FileStoreWrite::Create(seed_builder_load_options, std::move(seed_context), + FileStoreWrite::Create(seed_builder_load_context, std::move(seed_context), seed_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr seed_batch, MakeLogicalBatch({Row{99, "seed", "p0"}}, /*bucket=*/0)); @@ -5246,7 +5246,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) { ASSERT_OK_AND_ASSIGN(std::vector> seed_messages, seed_writer->PrepareCommit(/*wait_compaction=*/false, /*commit_identifier=*/0)); - TableLoadOptions seed_commit_builder_load_options{table_path_}; + TableLoadContext seed_commit_builder_load_context{table_path_}; std::map seed_commit_builder_options; CommitContextBuilder seed_commit_builder(commit_user_); seed_commit_builder_options = options_; @@ -5254,7 +5254,7 @@ TEST_F(RealtimeWriteInteTest, TestPkRecovery) { seed_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr seed_commit, - FileStoreCommit::Create(seed_commit_builder_load_options, std::move(seed_commit_context), + FileStoreCommit::Create(seed_commit_builder_load_context, std::move(seed_commit_context), seed_commit_builder_options)); ASSERT_OK(seed_commit->Commit(seed_messages, /*commit_identifier=*/0)); ASSERT_OK(seed_writer->Close()); @@ -5342,7 +5342,7 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context)); ASSERT_EQ(0, memory_usage); } - TableLoadOptions compact_builder_load_options{table_path_}; + TableLoadContext compact_builder_load_context{table_path_}; std::map compact_builder_options; WriteContextBuilder compact_builder(commit_user_); compact_builder_options = options_; @@ -5350,7 +5350,7 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { ASSERT_OK_AND_ASSIGN(std::unique_ptr compact_context, compact_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr compact_writer, - FileStoreWrite::Create(compact_builder_load_options, std::move(compact_context), + FileStoreWrite::Create(compact_builder_load_context, std::move(compact_context), compact_builder_options)); ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); @@ -5379,14 +5379,14 @@ TEST_F(RealtimeWriteInteTest, TestPkCompaction) { std::max(compacted_live_max_sequence_number, file->max_sequence_number); } ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber); - TableLoadOptions commit_builder_load_options{table_path_}; + TableLoadContext commit_builder_load_context{table_path_}; std::map commit_builder_options; CommitContextBuilder commit_builder(commit_user_); commit_builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); ASSERT_OK(commit->Commit(compact_messages, /*commit_identifier=*/4)); ASSERT_OK(compact_writer->Close()); @@ -5707,14 +5707,14 @@ TEST_F(RealtimeWriteInteTest, TestCommitWithProgressRejectsCoveredRangesFromAnot writer->PrepareCommitWithProgress(/*commit_identifier=*/0)); ASSERT_OK(Commit(commits, /*commit_identifier=*/0)); - TableLoadOptions builder_load_options{table_path_}; + TableLoadContext builder_load_context{table_path_}; std::map builder_options; CommitContextBuilder builder("another_realtime_commit_user"); builder_options = options_; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr commit, - FileStoreCommit::Create(builder_load_options, std::move(context), builder_options)); + FileStoreCommit::Create(builder_load_context, std::move(context), builder_options)); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(commits, /*commit_identifier=*/0, /*watermark=*/std::nullopt), "another commit user or identifier"); @@ -6530,7 +6530,7 @@ TEST_F(RealtimeWriteInteTest, TestSuccessfulReaderCreationConsumesRealtimeSplitT ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, CreatePlan(realtime_context, /*predicate=*/nullptr)); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; @@ -6539,7 +6539,7 @@ TEST_F(RealtimeWriteInteTest, TestSuccessfulReaderCreationConsumesRealtimeSplitT .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr first_reader, table_read->CreateReader(plan->Splits())); @@ -6565,7 +6565,7 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick CreatePlan(realtime_context, /*predicate=*/nullptr)); ASSERT_EQ(1, plan->Splits().size()); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; @@ -6576,7 +6576,7 @@ TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTick .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()), "cannot cast predicate unsupported"); @@ -7919,7 +7919,7 @@ TEST_F(RealtimeWriteInteTest, TestReaderPinsMemoryAcrossRefresh) { std::dynamic_pointer_cast(plan->Splits()[0]); ASSERT_NE(nullptr, realtime_split); - TableLoadOptions read_builder_load_options{table_path_}; + TableLoadContext read_builder_load_context{table_path_}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options = options_; @@ -7928,7 +7928,7 @@ TEST_F(RealtimeWriteInteTest, TestReaderPinsMemoryAcrossRefresh) { .WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 61a10e071..ca7c4970d 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -36,7 +36,7 @@ #include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -89,7 +89,7 @@ class ScanAndReadInteTest : public testing::Test, public: // Reuse the fixture's records with a schema declaring the engine under test. Merge engines // are immutable runtime options, so both operators must load that schema from the start. - Result LoadWithMergeEngine(const std::string& path, + Result LoadWithMergeEngine(const std::string& path, const std::string& merge_engine) const { SchemaManager manager(std::make_shared(), path); PAIMON_ASSIGN_OR_RAISE(std::optional> latest, @@ -100,7 +100,7 @@ class ScanAndReadInteTest : public testing::Test, PAIMON_ASSIGN_OR_RAISE( std::unique_ptr schema, latest.value()->MergeOptions({{Options::MERGE_ENGINE, merge_engine}})); - TableLoadOptions load{path}; + TableLoadContext load{path}; PAIMON_ASSIGN_OR_RAISE(load.table_schema_json, schema->GetJsonSchema()); return load; } @@ -263,7 +263,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -272,7 +272,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(scan_context_builder_load_options, std::move(scan_context).value(), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context).value(), scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); @@ -282,7 +282,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); @@ -290,7 +290,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { read_context_builder.Finish()); Result> table_read = - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = @@ -329,7 +329,7 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; @@ -338,7 +338,7 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { FinishScanContext(scan_context_builder, scan_context_builder_options); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = - TableScan::Create(scan_context_builder_load_options, std::move(scan_context).value(), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context).value(), scan_context_builder_options); CHECK_HOOK_STATUS(table_scan.status(), i); Result> result_plan = table_scan.value()->CreatePlan(); @@ -347,7 +347,7 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { auto splits = result_plan.value()->Splits(); ASSERT_EQ(3, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); @@ -355,7 +355,7 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); Result> table_read = - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = @@ -391,7 +391,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -399,7 +399,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -407,14 +407,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -443,7 +443,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; @@ -451,7 +451,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -460,13 +460,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { ASSERT_EQ(3, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -566,7 +566,7 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { bool entry_cache) -> Result { std::shared_ptr pool = GetMemoryPool(); scan_pools.push_back(pool); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = entry_cache ? "3" : "0"; @@ -581,7 +581,7 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_scan, - TableScan::Create(builder_load_options, std::move(context), builder_options)); + TableScan::Create(builder_load_context, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); PAIMON_ASSIGN_OR_RAISE(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); @@ -599,14 +599,14 @@ TEST(SelectiveManifestDecodeInteTest, TestPartitionedPointLookup) { (historical && data_split->Bucket() == kBuckets / 2 - 1)); ASSERT_EQ(data_split->GetFileList().size(), 1); } - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ReadContextBuilder builder; builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, - TableRead::Create(builder_load_options, std::move(context), builder_options)); + TableRead::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(result.plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, @@ -755,7 +755,7 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) } auto cache = std::make_shared(capacities); for (int32_t attempt = 0; attempt < 2; ++attempt) { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS] = "3"; @@ -765,7 +765,7 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(builder_load_options, std::move(context), builder_options)); + TableScan::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -774,7 +774,7 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) ASSERT_OK_AND_ASSIGN(uint64_t hit, scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); ASSERT_EQ(hit, attempt); - TableLoadOptions read_builder_load_options{table_path}; + TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder.SetPredicate(predicate).EnablePredicateFilter(true); @@ -782,7 +782,7 @@ TEST(SelectiveManifestDecodeInteTest, TestSchemaEvolutionPreservesBucketPruning) read_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); @@ -850,12 +850,12 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { // All buckets have overlapping stats for this key. An explicit filter bypasses // inference, proving that ordinary statistics cannot account for the pruning. for (int32_t bucket = 0; bucket < kNumBuckets; ++bucket) { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder.SetPredicate(predicate).SetBucketFilter(bucket); ASSERT_OK_AND_ASSIGN(auto context, FinishScanContext(builder, builder_options)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(builder_load_options, + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); @@ -878,7 +878,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); - TableLoadOptions scan_builder_load_options{table_path}; + TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options[Options::BUCKET] = current_bucket_count; @@ -886,7 +886,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder, scan_builder_options)); ASSERT_OK_AND_ASSIGN( - auto scan, TableScan::Create(scan_builder_load_options, std::move(scan_context), + auto scan, TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { @@ -894,7 +894,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { uint64_t enabled, scan->GetMetrics()->GetCounter(ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); - TableLoadOptions cached_builder_load_options{table_path}; + TableLoadContext cached_builder_load_context{table_path}; std::map cached_builder_options; ScanContextBuilder cached_builder; cached_builder_options[Options::BUCKET] = current_bucket_count; @@ -903,7 +903,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { FinishScanContext(cached_builder, cached_builder_options)); ASSERT_OK_AND_ASSIGN( auto cached_scan, - TableScan::Create(cached_builder_load_options, std::move(cached_context), + TableScan::Create(cached_builder_load_context, std::move(cached_context), cached_builder_options)); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( @@ -917,14 +917,14 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { ASSERT_TRUE(data_split); ASSERT_EQ(data_split->Bucket(), bucket_ids[lookup_index]); } - TableLoadOptions read_builder_load_options{table_path}; + TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; ReadContextBuilder read_builder; AddReadOptionsForPrefetch(&read_builder, read_builder_options); read_builder.SetPredicate(lookup_predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto read, TableRead::Create(read_builder_load_options, std::move(read_context), + auto read, TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, @@ -949,7 +949,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; @@ -957,7 +957,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -966,13 +966,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { ASSERT_EQ(3, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -1009,7 +1009,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5WithReaderBuildParallelism) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; @@ -1017,19 +1017,19 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5WithReaderBuildParallelism) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -1061,7 +1061,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1069,16 +1069,16 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 1, 2, 3, 4}; @@ -1120,7 +1120,7 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { std::string table_path = GetDataDir() + "/" + file_format + "/append_10.db/append_10"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1128,7 +1128,7 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1137,13 +1137,13 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { ASSERT_EQ(3, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -1169,7 +1169,7 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { "/append_complex_build_in_fieldid.db/" "append_complex_build_in_fieldid/"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1177,7 +1177,7 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1203,13 +1203,13 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { std::shared_ptr arrow_data_type = DataField::ConvertDataFieldsToArrowStructType(read_fields); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -1234,7 +1234,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // normal batch scan case for pk+dv, all data in level 0 is filtered out - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; @@ -1242,15 +1242,15 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1285,7 +1285,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1293,16 +1293,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1318,7 +1318,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; @@ -1327,16 +1327,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1364,7 +1364,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // predicate: f0 != "Alice" (key predicate) and f3 > 18 (value predicate) and all data in level // 0 - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; @@ -1380,17 +1380,17 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1414,7 +1414,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; @@ -1430,17 +1430,17 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLateMaterializ ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1464,7 +1464,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; @@ -1477,7 +1477,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1492,23 +1492,23 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { // normal batch scan case for pk+dv, all data in level 0 is filtered out, set row limits to 6, // data in partition 20 is truncated - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(6); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1537,7 +1537,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; @@ -1546,16 +1546,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // from an compact snapshot @@ -1598,7 +1598,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; @@ -1607,16 +1607,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, second plan is snapshot6 @@ -1653,7 +1653,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; @@ -1661,16 +1661,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot6 with merge, level 0 data in base manifest @@ -1700,7 +1700,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1709,16 +1709,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // from the first snapshot @@ -1768,7 +1768,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; @@ -1777,16 +1777,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // use from-snapshot mode, first plan is empty, second plan is snapshot 3 with delta files @@ -1822,16 +1822,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_nested_type.db/pk_table_nested_type/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); @@ -1841,7 +1841,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { .Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1885,22 +1885,22 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // normal batch scan case for pk+mor, use latest snapshot if not specified - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1941,7 +1941,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { // normal batch scan case for pk+mor, read snapshot 2 (append snapshot), all data is in level 0 // with merge read - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; @@ -1949,16 +1949,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1995,7 +1995,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // all data not in partition 10, bucket 1 is filtered out - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; @@ -2004,16 +2004,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2043,7 +2043,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; @@ -2070,17 +2070,17 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(false); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2115,7 +2115,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; @@ -2136,17 +2136,17 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLateMateriali ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnableLateMaterializing(true); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2172,7 +2172,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; @@ -2186,7 +2186,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2199,7 +2199,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - ASSERT_OK_AND_ASSIGN(TableLoadOptions scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(TableLoadContext scan_context_builder_load_context, LoadWithMergeEngine(table_path, "aggregation")); std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; @@ -2210,10 +2210,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options = scan_context_builder_load_options; + TableLoadContext read_context_builder_load_context = scan_context_builder_load_context; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); @@ -2221,7 +2221,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2249,7 +2249,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ASSERT_OK_AND_ASSIGN(TableLoadOptions scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(TableLoadContext scan_context_builder_load_context, LoadWithMergeEngine(table_path, "aggregation")); std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; @@ -2260,10 +2260,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options = scan_context_builder_load_options; + TableLoadContext read_context_builder_load_context = scan_context_builder_load_context; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); @@ -2271,7 +2271,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2286,7 +2286,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ASSERT_OK_AND_ASSIGN(TableLoadOptions scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(TableLoadContext scan_context_builder_load_context, LoadWithMergeEngine(table_path, "aggregation")); std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; @@ -2301,10 +2301,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options = scan_context_builder_load_options; + TableLoadContext read_context_builder_load_context = scan_context_builder_load_context; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); @@ -2312,7 +2312,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2340,7 +2340,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - ASSERT_OK_AND_ASSIGN(TableLoadOptions scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(TableLoadContext scan_context_builder_load_context, LoadWithMergeEngine(table_path, "partial-update")); std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; @@ -2355,10 +2355,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options = scan_context_builder_load_options; + TableLoadContext read_context_builder_load_context = scan_context_builder_load_context; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); @@ -2366,7 +2366,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2396,23 +2396,23 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { // in pk mor mode, limit does not take effect, as we do not know the number of records after // merging - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(6); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2445,7 +2445,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; @@ -2454,16 +2454,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // first plan is snapshot4 (isStreaming=false), second plan is snapshot5 @@ -2499,7 +2499,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -2508,16 +2508,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {1, 2, 4, 5}; @@ -2569,7 +2569,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { // use from-snapshot mode, first plan is empty, snapshot 2 in second plan only includes delta // files - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; @@ -2578,16 +2578,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 2, 4, 5}; @@ -2623,7 +2623,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; @@ -2635,17 +2635,17 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // first plan is snapshot 5 with empty plan, second plan is snapshot 5 with delta files @@ -2668,7 +2668,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamPredicateFilter) { std::string table_path = paimon::test::GetDataDir() + FileFormat() + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; @@ -2680,17 +2680,17 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamPredicateFilter) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate).EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); std::vector> expected_snapshot_ids = {std::nullopt, 5}; @@ -2713,7 +2713,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; // normal batch scan case for pk+first row, all data in level 0 is filtered out - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "5"; @@ -2721,16 +2721,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2763,7 +2763,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; @@ -2772,16 +2772,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // from a compact snapshot @@ -2825,7 +2825,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_MODE] = "latest-full"; @@ -2833,16 +2833,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // from an append snapshot, first plan is snapshot5 with merge, all level 0 data is merged @@ -2874,23 +2874,23 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_09.db/pk_09/"; // normal batch scan case for pk+dv (09 version) - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetLimit(2); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2924,22 +2924,22 @@ TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { const std::vector>& partition_filters, const std::shared_ptr& expected) { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetPartitionFilter(partition_filters); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -3001,22 +3001,22 @@ TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { auto check_result = [&](const std::vector>& partition_filters, const std::shared_ptr& expected) { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.SetPartitionFilter(partition_filters); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -3058,7 +3058,7 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_09.db/append_09/"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -3066,7 +3066,7 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3076,14 +3076,14 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { // read std::shared_ptr read_pool = GetMemoryPool(); { - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.WithMemoryPool(read_pool); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); @@ -3169,13 +3169,13 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { { // batch scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder.WithStreamingMode(false); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -3184,24 +3184,24 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot 1 - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; scan_context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -3228,24 +3228,24 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot 2 - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; scan_context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -3264,7 +3264,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } { // stream scan: from snapshot full - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -3272,16 +3272,16 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { scan_context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -3315,7 +3315,7 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar } std::string table_path = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; @@ -3325,20 +3325,20 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); ASSERT_EQ(result_plan->Splits().size(), 1); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"f10", "f8", "f4", "f13"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3375,7 +3375,7 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { auto predicate = PredicateBuilder::GreaterThan( /*field_index=*/2, /*field_name=*/"key_2", FieldType::INT, Literal(500)); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; @@ -3384,7 +3384,7 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -3398,14 +3398,14 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { } ASSERT_EQ(data_file_count, 1); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3438,7 +3438,7 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { paimon::test::GetDataDir() + "/append_table_with_multiple_file_format.db/append_table_with_multiple_file_format"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "2"; @@ -3446,7 +3446,7 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -3454,13 +3454,13 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3487,13 +3487,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_no_external.db/pk_dv_index_in_data_no_external"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3501,13 +3501,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3536,13 +3536,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_no_external.db/pk_dv_index_not_in_data_no_external"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3550,13 +3550,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { auto splits = result_plan->Splits(); ASSERT_EQ(3, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3587,13 +3587,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_with_external.db/external"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3603,13 +3603,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/false, &splits); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3641,13 +3641,13 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { Literal(FieldType::STRING, "Alice0", 6)); // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options["file-index.read.enabled"] = "false"; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3655,7 +3655,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { auto splits = result_plan->Splits(); ASSERT_EQ(1, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options["file-index.read.enabled"] = "false"; @@ -3663,7 +3663,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3724,7 +3724,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(15)); - TableLoadOptions stats_only_scan_builder_load_options{table_path}; + TableLoadContext stats_only_scan_builder_load_context{table_path}; std::map stats_only_scan_builder_options; ScanContextBuilder stats_only_scan_builder; stats_only_scan_builder_options = options; @@ -3734,13 +3734,13 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr stats_only_scan, - TableScan::Create(stats_only_scan_builder_load_options, std::move(stats_only_scan_context), + TableScan::Create(stats_only_scan_builder_load_context, std::move(stats_only_scan_context), stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // The embedded bitmap knows that 15 is absent and eliminates the file during scan planning. - TableLoadOptions indexed_scan_builder_load_options{table_path}; + TableLoadContext indexed_scan_builder_load_context{table_path}; std::map indexed_scan_builder_options; ScanContextBuilder indexed_scan_builder; indexed_scan_builder_options = options; @@ -3749,7 +3749,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { indexed_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr indexed_scan, - TableScan::Create(indexed_scan_builder_load_options, std::move(indexed_scan_context), + TableScan::Create(indexed_scan_builder_load_context, std::move(indexed_scan_context), indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); @@ -3759,7 +3759,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { // the other rows. const std::shared_ptr present_predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"indexed_value", FieldType::INT, Literal(20)); - TableLoadOptions retained_scan_builder_load_options{table_path}; + TableLoadContext retained_scan_builder_load_context{table_path}; std::map retained_scan_builder_options; ScanContextBuilder retained_scan_builder; retained_scan_builder_options = options; @@ -3768,19 +3768,19 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { retained_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr retained_scan, - TableScan::Create(retained_scan_builder_load_options, std::move(retained_scan_context), + TableScan::Create(retained_scan_builder_load_context, std::move(retained_scan_context), retained_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr retained_plan, retained_scan->CreatePlan()); ASSERT_FALSE(retained_plan->Splits().empty()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetPredicate(present_predicate).EnablePredicateFilter(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(retained_plan->Splits())); @@ -3841,7 +3841,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); // The evolved min/max range is still [10, 30], so statistics alone retain the old file. - TableLoadOptions stats_only_scan_builder_load_options{table_path}; + TableLoadContext stats_only_scan_builder_load_context{table_path}; std::map stats_only_scan_builder_options; ScanContextBuilder stats_only_scan_builder; stats_only_scan_builder_options = options; @@ -3851,13 +3851,13 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr stats_only_scan, - TableScan::Create(stats_only_scan_builder_load_options, std::move(stats_only_scan_context), + TableScan::Create(stats_only_scan_builder_load_context, std::move(stats_only_scan_context), stats_only_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); // Reconstructing the predicate to schema-0 lets its embedded bitmap prove that 15 is absent. - TableLoadOptions indexed_scan_builder_load_options{table_path}; + TableLoadContext indexed_scan_builder_load_context{table_path}; std::map indexed_scan_builder_options; ScanContextBuilder indexed_scan_builder; indexed_scan_builder_options = options; @@ -3866,7 +3866,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc indexed_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr indexed_scan, - TableScan::Create(indexed_scan_builder_load_options, std::move(indexed_scan_context), + TableScan::Create(indexed_scan_builder_load_context, std::move(indexed_scan_context), indexed_scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_plan, indexed_scan->CreatePlan()); ASSERT_TRUE(indexed_plan->Splits().empty()); @@ -3880,13 +3880,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { std::string external_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_with_external.db/external"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -3896,13 +3896,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { AdjustSplitWithExternalPath("FILE:/tmp/external", external_path, /*adjust_index=*/true, &splits); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3930,13 +3930,13 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_multiple_ts_precision_and_timezone.db" "/append_with_multiple_ts_precision_and_timezone/"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3945,13 +3945,13 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { ASSERT_EQ(1, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3987,13 +3987,13 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { "/append_with_cast_timestamp.db" "/append_with_cast_timestamp/"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -4002,13 +4002,13 @@ TEST_P(ScanAndReadInteTest, TestCastTimestampType) { ASSERT_EQ(1, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -4044,23 +4044,23 @@ TEST_F(ScanAndReadInteTest, TestParquetDictionaryBinaryStoredArrowSchema) { "/parquet/append_dictionary_binary.db/" "append_dictionary_binary"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->SnapshotId().value()); ASSERT_EQ(1, plan->Splits().size()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); @@ -4130,10 +4130,10 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { const std::vector& projection, const std::shared_ptr& expected_result, bool exact_filter = false) { - TableLoadOptions scan_builder_load_options{table_path}; + TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; ScanContextBuilder scan_builder; - TableLoadOptions read_builder_load_options{table_path}; + TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; ReadContextBuilder read_builder; if (predicate) { @@ -4151,14 +4151,14 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(2)); ASSERT_FALSE(plan->Splits().empty()); ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -4230,20 +4230,20 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { auto check = [&](int64_t snapshot_id, const std::vector& projection, const std::shared_ptr& expected_result) { SCOPED_TRACE("snapshot " + std::to_string(snapshot_id)); - TableLoadOptions scan_builder_load_options{table_path}; + TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(snapshot_id)); ASSERT_FALSE(plan->Splits().empty()); - TableLoadOptions read_builder_load_options{table_path}; + TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options["read.batch-size"] = "1"; @@ -4253,7 +4253,7 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { } ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -4314,20 +4314,20 @@ TEST_F(ScanAndReadInteTest, TestLanceLtzCompatibility) { const std::vector& projection, const std::shared_ptr& expected_result) { SCOPED_TRACE("snapshot " + std::to_string(snapshot_id)); - TableLoadOptions scan_builder_load_options{table_path}; + TableLoadContext scan_builder_load_context{table_path}; std::map scan_builder_options; ScanContextBuilder scan_builder; scan_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto scan, - TableScan::Create(scan_builder_load_options, std::move(scan_context), + TableScan::Create(scan_builder_load_context, std::move(scan_context), scan_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId(), std::optional(snapshot_id)); ASSERT_FALSE(plan->Splits().empty()); - TableLoadOptions read_builder_load_options{table_path}; + TableLoadContext read_builder_load_context{table_path}; std::map read_builder_options; ReadContextBuilder read_builder; read_builder_options["read.batch-size"] = "1"; @@ -4337,7 +4337,7 @@ TEST_F(ScanAndReadInteTest, TestLanceLtzCompatibility) { } ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto read, - TableRead::Create(read_builder_load_options, std::move(read_context), + TableRead::Create(read_builder_load_context, std::move(read_context), read_builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -4406,10 +4406,10 @@ TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) { auto check_compatibility = [](const std::string& table_path, const std::shared_ptr& predicate, const std::shared_ptr& expected_result) { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; if (predicate != nullptr) { @@ -4420,7 +4420,7 @@ TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) { scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); @@ -4428,7 +4428,7 @@ TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) { read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); @@ -4478,20 +4478,20 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibility) { const std::string table_name = "append_java_compat"; const std::string table_path = GetDataDir() + "/vortex/" + table_name + ".db/" + table_name; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); @@ -4512,24 +4512,24 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityLtzAcrossTimezones) { const std::string timezones[] = {"UTC", "Asia/Shanghai"}; for (const std::string& tz : timezones) { TimezoneGuard timezone_guard(tz); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, std::move(scan_context), + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, std::move(read_context), + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); @@ -4595,20 +4595,20 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityAllTypes) { const std::string table_name = "all_types_java_compat"; const std::string table_path = GetDataDir() + "/vortex/" + table_name + ".db/" + table_name; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); @@ -4624,13 +4624,13 @@ TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) { auto read_data = [](int64_t snapshot_id, const std::string& result_json) { std::string table_path = GetDataDir() + "/avro/append_multiple.db/append_multiple"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -4639,7 +4639,7 @@ TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) { ASSERT_EQ(3, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options["test.enable-adaptive-prefetch-strategy"] = "false"; @@ -4647,7 +4647,7 @@ TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) { read_context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); @@ -4709,13 +4709,13 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { std::string table_path = GetDataDir() + "/avro/pk_with_multiple_type.db/pk_with_multiple_type"; // scan - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -4725,12 +4725,12 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { ASSERT_EQ(1, splits.size()); // read - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); @@ -4791,7 +4791,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { // Historical files use two buckets even when the current option has changed. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; @@ -4802,17 +4802,17 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -4821,7 +4821,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { ASSERT_OK_AND_ASSIGN(uint64_t enabled, table_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_ENABLED)); ASSERT_EQ(enabled, 1); - TableLoadOptions cached_builder_load_options{table_path}; + TableLoadContext cached_builder_load_context{table_path}; std::map cached_builder_options; ScanContextBuilder cached_builder; cached_builder_options[Options::SCAN_SNAPSHOT_ID] = "6"; @@ -4829,7 +4829,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { cached_builder.SetPartitionFilter({{{"f1", "10"}}}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder, cached_builder_options)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(cached_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(cached_builder_load_context, std::move(cached_context), cached_builder_options)); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); @@ -4879,26 +4879,26 @@ TEST_P(ScanAndReadInteTest, TestReadNullableMapKey) { const std::string table_path = paimon::test::GetDataDir() + "orc/nullable_map_key.db/nullable_map_key"; - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->SnapshotId()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_EQ(result_plan->Splits().size(), 1); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options[Options::READ_BATCH_SIZE] = "1"; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -4931,13 +4931,13 @@ TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); // CreateCountReader with empty splits should return 0 rows. @@ -4953,38 +4953,38 @@ TEST_P(ScanAndReadInteTest, TestCountRowsConsistencyWithCreateReader) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Scan latest snapshot - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Method 1: CreateCountReader + iterate batches - TableLoadOptions count_context_builder_load_options{table_path}; + TableLoadContext count_context_builder_load_context{table_path}; std::map count_context_builder_options; ReadContextBuilder count_context_builder; AddReadOptionsForPrefetch(&count_context_builder, count_context_builder_options); ASSERT_OK_AND_ASSIGN(auto count_read_context, count_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto count_table_read, - TableRead::Create(count_context_builder_load_options, std::move(count_read_context), + TableRead::Create(count_context_builder_load_context, std::move(count_read_context), count_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto count_reader, count_table_read->CreateCountReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(int64_t count_result, count_reader->CountRows()); // Method 2: CreateReader + iterate batches - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -5001,27 +5001,27 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithPredicateNotSupported) { "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // Set predicate in read context. CountReader currently does not support predicate pushdown. auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(result_plan->Splits()), @@ -5034,23 +5034,23 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithForceKeepDeleteNotSupported "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder, scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; AddReadOptionsForPrefetch(&read_context_builder, read_context_builder_options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); auto* key_value_table_read = dynamic_cast(table_read.get()); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index d01a79480..accfaf7f5 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -32,7 +32,7 @@ #include "arrow/api.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/decimal_utils.h" @@ -271,14 +271,14 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; auto run_scan = [&]() -> Result> { - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; context_builder.WithCache(manifest_cache); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_scan, - TableScan::Create(context_builder_load_options, + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); return table_scan->CreatePlan(); }; @@ -300,7 +300,7 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { Result>> RunBucketSnapshotScan( const std::string& table_path, int64_t snapshot_id, int32_t bucket, const std::shared_ptr& cache, int32_t max_snapshot_live_manifest_versions) { - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); @@ -315,7 +315,7 @@ Result>> RunBucketSnapshotScan( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); @@ -407,7 +407,7 @@ TEST(ScanInteManifestCacheTest, TestSnapshotLiveManifestCacheRetainsSnapshotsPer TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -415,7 +415,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -469,7 +469,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; @@ -477,7 +477,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -536,18 +536,18 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { std::string table_path = data_dir + "orc/append_09.db/append_09"; auto check_result = [&](const std::optional& specific_table_schema) { - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; if (specific_table_schema) { - context_builder_load_options.table_schema_json = specific_table_schema.value(); + context_builder_load_context.table_schema_json = specific_table_schema.value(); } ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -606,7 +606,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { TEST_P(ScanInteTest, TestScanInvalidSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "100"; @@ -614,7 +614,7 @@ TEST_P(ScanInteTest, TestScanInvalidSnapshot) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), @@ -623,7 +623,7 @@ TEST_P(ScanInteTest, TestScanInvalidSnapshot) { TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -631,7 +631,7 @@ TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // batch scan multiple @@ -640,7 +640,7 @@ TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; @@ -651,7 +651,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -716,7 +716,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; @@ -724,7 +724,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -768,7 +768,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "3"; @@ -776,7 +776,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -817,7 +817,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { // from snapshot is specified std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -825,7 +825,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1_1( @@ -935,7 +935,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); @@ -943,7 +943,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1( @@ -994,7 +994,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); @@ -1002,7 +1002,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1056,7 +1056,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_MODE] = StartupMode::Latest().ToString(); @@ -1064,7 +1064,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); std::vector> expected_snapshot_ids = {}; @@ -1074,7 +1074,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); @@ -1083,7 +1083,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder2_1( @@ -1151,7 +1151,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshotFull().ToString(); @@ -1161,7 +1161,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { FinishScanContext(context_builder, context_builder_options)); ASSERT_TRUE(scan_context); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1_1( @@ -1241,7 +1241,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; { - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); @@ -1249,21 +1249,21 @@ TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::BUCKET] = "-2"; context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); - ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), + ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options), "do not support bucket=-2 in scan process"); } @@ -1277,7 +1277,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, val.data(), val.size())); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1285,7 +1285,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1329,7 +1329,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { PredicateBuilder::And({greater_than_predicate, less_than_predicate, not_in_predicate, not_equal})); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1337,7 +1337,7 @@ TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); DataSplitImpl::Builder builder1_2( @@ -1394,7 +1394,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { std::map partition_keys; partition_keys["f1"] = "10"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1402,7 +1402,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1446,14 +1446,14 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithInvalidPartitionFilter) { std::map partition_keys; partition_keys["invalid_partition_key"] = "10"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; context_builder.SetPartitionFilter({partition_keys}); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); - ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), + ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options), "field invalid_partition_key does not exist in partition keys"); } @@ -1468,7 +1468,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF std::map partition_keys; partition_keys["f1"] = "10"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1476,7 +1476,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1524,7 +1524,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { partition_keys["f1"] = "10"; partition_keys["f2"] = "0"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1532,7 +1532,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1586,7 +1586,7 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) Literal(paimon::Decimal(23, 5, 0))); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; @@ -1594,7 +1594,7 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan); @@ -1653,7 +1653,7 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) Literal(FieldType::BINARY, "zoo", 3)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "4"; @@ -1661,7 +1661,7 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1714,7 +1714,7 @@ TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", @@ -1724,7 +1724,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1817,7 +1817,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", @@ -1831,7 +1831,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1869,7 +1869,7 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_SNAPSHOT_ID] = "1"; @@ -1877,7 +1877,7 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1887,14 +1887,14 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::BUCKET] = "-1"; context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); - ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_options, std::move(scan_context), + ASSERT_NOK_WITH_MSG(TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options), "Cannot change bucket from or to -1 dynamically"); } @@ -1902,13 +1902,13 @@ TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { TEST_P(ScanInteTest, TestReadWithNoSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_nested_type.db/append_table_with_nested_type"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->SnapshotId()); @@ -1919,7 +1919,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_alter_table_with_cast.db/append_table_alter_table_with_cast"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; @@ -1948,7 +1948,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1990,7 +1990,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table.db/append_table_with_alter_table"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; @@ -2005,7 +2005,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2074,7 +2074,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table_with_dense_field.db/" "append_table_with_alter_table_with_dense_field"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; @@ -2089,7 +2089,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2128,7 +2128,7 @@ TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; @@ -2143,7 +2143,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2211,7 +2211,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; @@ -2226,7 +2226,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); @@ -2236,7 +2236,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; @@ -2251,7 +2251,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -2290,7 +2290,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; // file0 will be removed as f5 not exists @@ -2302,7 +2302,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2365,7 +2365,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; auto child1 = PredicateBuilder::IsNull(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT); @@ -2378,7 +2378,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2446,7 +2446,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; @@ -2461,7 +2461,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2529,7 +2529,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; @@ -2546,7 +2546,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); @@ -2555,7 +2555,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { TEST_P(ScanInteTest, TestScanAppendWithTag1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_TAG_NAME] = "1"; @@ -2563,7 +2563,7 @@ TEST_P(ScanInteTest, TestScanAppendWithTag1) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -2573,7 +2573,7 @@ TEST_P(ScanInteTest, TestScanAppendWithTag1) { TEST_P(ScanInteTest, TestScanInvalidTag) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ScanContextBuilder context_builder; context_builder_options[Options::SCAN_TAG_NAME] = "unknown"; @@ -2581,7 +2581,7 @@ TEST_P(ScanInteTest, TestScanInvalidTag) { ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder, context_builder_options)); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(context_builder_load_options, std::move(scan_context), + TableScan::Create(context_builder_load_context, std::move(scan_context), context_builder_options)); ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "Tag 'unknown' doesn't exist."); } @@ -2595,7 +2595,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { // EarlierOrEqual(snap3.time) → snap-3 { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); @@ -2604,13 +2604,13 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { FinishScanContext(builder, builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(builder_load_options, std::move(ctx), builder_options)); + TableScan::Create(builder_load_context, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 3); } // EarlierOrEqual(snap3.time - 1) → snap-2 { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis() - 1); @@ -2619,13 +2619,13 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { FinishScanContext(builder, builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(builder_load_options, std::move(ctx), builder_options)); + TableScan::Create(builder_load_context, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 2); } // EarlierOrEqual(INT64_MAX) → snap-5 (latest) { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_TIMESTAMP_MILLIS] = @@ -2635,13 +2635,13 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { FinishScanContext(builder, builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(builder_load_options, std::move(ctx), builder_options)); + TableScan::Create(builder_load_context, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 5); } // EarlierOrEqual(0) → no snapshot found, expect error { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; @@ -2650,7 +2650,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { FinishScanContext(builder, builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(builder_load_options, std::move(ctx), builder_options)); + TableScan::Create(builder_load_context, std::move(ctx), builder_options)); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "There is currently no snapshot earlier than or equal to timestamp"); } @@ -2666,7 +2666,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { // T=0: no snapshot earlier than T=0, stream starts from snapshot 1. { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_TIMESTAMP_MILLIS] = "0"; @@ -2675,7 +2675,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { FinishScanContext(builder, builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(builder_load_options, std::move(ctx), builder_options)); + TableScan::Create(builder_load_context, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2683,7 +2683,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap2.time+1: EarlierThan(snap2.time+1)=snap2, stream starts from snapshot 3. { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap2.TimeMillis() + 1); @@ -2692,7 +2692,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { FinishScanContext(builder, builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(builder_load_options, std::move(ctx), builder_options)); + TableScan::Create(builder_load_context, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2700,7 +2700,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=snap3.time: EarlierThan(snap3.time)=snap2, stream starts from snapshot 3. { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_TIMESTAMP_MILLIS] = std::to_string(snap3.TimeMillis()); @@ -2709,7 +2709,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { FinishScanContext(builder, builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(builder_load_options, std::move(ctx), builder_options)); + TableScan::Create(builder_load_context, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan1, scan->CreatePlan()); @@ -2717,7 +2717,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } // T=INT64_MAX: EarlierThan(INT64_MAX)=snap5, stream starts from snapshot 6 (beyond latest). { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; builder_options[Options::SCAN_TIMESTAMP_MILLIS] = @@ -2727,7 +2727,7 @@ TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { FinishScanContext(builder, builder_options)); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, - TableScan::Create(builder_load_options, std::move(ctx), builder_options)); + TableScan::Create(builder_load_context, std::move(ctx), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); ASSERT_TRUE(plan0->Splits().empty()); diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 278c81a51..cd829d3c3 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -35,7 +35,7 @@ #include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/reader/reader_utils.h" @@ -171,7 +171,7 @@ class WriteAndReadInteTest const std::map& partition, const std::set& buckets, bool full_compaction, int64_t commit_identifier) const { - TableLoadOptions write_context_builder_load_options{table_path}; + TableLoadContext write_context_builder_load_context{table_path}; std::map write_context_builder_options; WriteContextBuilder write_context_builder("commit_user"); @@ -181,7 +181,7 @@ class WriteAndReadInteTest write_context_builder.WithStreamingMode(true).Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_store_write, - FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_context_builder_load_context, std::move(write_context), write_context_builder_options)); for (int32_t bucket : buckets) { PAIMON_RETURN_NOT_OK(file_store_write->Compact(partition, bucket, full_compaction)); @@ -193,7 +193,7 @@ class WriteAndReadInteTest if (compact_messages.empty()) { return Status::Invalid("expected compaction commit messages"); } - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user"); commit_context_builder_options = options; @@ -201,7 +201,7 @@ class WriteAndReadInteTest commit_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_store_commit, - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); return file_store_commit->Commit(compact_messages, commit_identifier); } @@ -213,13 +213,13 @@ class WriteAndReadInteTest std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadFieldNames(read_fields); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); @@ -241,13 +241,13 @@ class WriteAndReadInteTest auto c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_schema.get())); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadSchema(std::move(c_schema)); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); @@ -278,14 +278,14 @@ class WriteAndReadInteTest Result> InnerScan( const std::map& options) const { std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); return table_scan->CreatePlan(); @@ -485,7 +485,7 @@ TEST_P(WriteAndReadInteTest, TestAppendReadWithNestedPredicateAcrossBatches) { SCOPED_TRACE(batch_size); for (bool enable_filter : {false, true}) { SCOPED_TRACE(enable_filter); - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ReadContextBuilder builder; builder_options = options; @@ -498,7 +498,7 @@ TEST_P(WriteAndReadInteTest, TestAppendReadWithNestedPredicateAcrossBatches) { ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); ASSERT_OK_AND_ASSIGN( auto table_read, - TableRead::Create(builder_load_options, std::move(context), builder_options)); + TableRead::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); ASSERT_TRUE(actual); @@ -664,7 +664,7 @@ TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(2)); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -672,19 +672,19 @@ TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) { scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -755,7 +755,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithExternalBitmapAndRangeBitmapIndexes) ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({name_predicate, score_predicate})); std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -763,21 +763,21 @@ TEST_P(WriteAndReadInteTest, TestAppendWithExternalBitmapAndRangeBitmapIndexes) scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(1, plan->Splits().size()); // Keep precise post-read filtering disabled. The exact result therefore verifies that the // bitmap and range-bitmap indexes produced by the write path are consumed by the read path. - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -876,7 +876,7 @@ TEST_P(WriteAndReadInteTest, TestLuminaSearchAfterRenameAcrossSchemaVersions) { auto search = std::make_shared("vec", 3, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder_options = current_options; @@ -885,7 +885,7 @@ TEST_P(WriteAndReadInteTest, TestLuminaSearchAfterRenameAcrossSchemaVersions) { .EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); ASSERT_OK_AND_ASSIGN( - auto read, TableRead::Create(builder_load_options, std::move(context), builder_options)); + auto read, TableRead::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), @@ -977,7 +977,7 @@ TEST_P(WriteAndReadInteTest, TestLuminaVectorIndexAfterCompaction) { auto search = std::make_shared( "embedding", limit, std::vector{0.0f, 0.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder_options = options; @@ -987,7 +987,7 @@ TEST_P(WriteAndReadInteTest, TestLuminaVectorIndexAfterCompaction) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr read, - TableRead::Create(builder_load_options, std::move(context), builder_options)); + TableRead::Create(builder_load_context, std::move(context), builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr actual, @@ -1102,7 +1102,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithLuminaVectorIndex) { "embedding", /*limit=*/3, std::vector{0.0f, 0.0f}, /*pre_filter=*/nullptr, /*predicate=*/nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - TableLoadOptions builder_load_options{PathUtil::JoinPath(root, "foo.db/bar")}; + TableLoadContext builder_load_context{PathUtil::JoinPath(root, "foo.db/bar")}; std::map builder_options; ReadContextBuilder builder; builder.SetReadFieldNames({"id", "_embedding_INDEX_SCORE"}) @@ -1111,7 +1111,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithLuminaVectorIndex) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, - TableRead::Create(builder_load_options, std::move(context), builder_options)); + TableRead::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -3368,14 +3368,14 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -3385,13 +3385,13 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { // of the result is attributable to split/file/RG/page pruning, not to a // post-read row-level filter. This is what makes the exact assertion below // meaningful as a check that page-index filtering is wired and working. - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3486,14 +3486,14 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -3502,13 +3502,13 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { // Read with predicate but WITHOUT EnablePredicateFilter, so the narrowing // observed below is attributable to page-index filtering rather than a // post-read row-level filter. - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3580,7 +3580,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { {Literal(7), Literal(11), Literal(15)}); ASSERT_TRUE(predicate); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -3588,7 +3588,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); @@ -3598,7 +3598,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { // therefore finds no data for it and skips ahead into the next row group it owns, whose first // selected row sits in the middle of that row group. Row level filtering stays off: the // expected rows below are exactly what page-index filtering selects. - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; @@ -3609,7 +3609,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { .SetPrefetchBatchCount(3); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto read_result, @@ -3677,13 +3677,13 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { auto cache = std::make_shared(CacheKind::DATA_FILE_FOOTER, 128 * 1024 * 1024); auto read_once = [&]() -> Result { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::SCAN_MODE] = StartupMode::LatestFull().ToString(); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); @@ -3694,12 +3694,12 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { return Status::Invalid("no splits found"); } - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder.WithCache(cache); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -4008,7 +4008,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { (void)helper; std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); - TableLoadOptions write_context_builder_load_options{table_path}; + TableLoadContext write_context_builder_load_context{table_path}; std::map write_context_builder_options; WriteContextBuilder write_context_builder("commit_user_1"); write_context_builder_options = options; @@ -4016,7 +4016,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { write_context_builder.Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(write_context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(write_context_builder_load_context, std::move(write_context), write_context_builder_options)); auto write_one_row = [&](const std::string& data) -> Status { @@ -4034,20 +4034,20 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { /*commit_identifier=*/0)); ASSERT_OK(file_store_write->Close()); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options)); ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(10)); - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -4055,19 +4055,19 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->Splits().empty()); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -5125,13 +5125,13 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionRea auto read_fields = [&](const std::vector& field_names) -> Status { PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadFieldNames(field_names); PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); @@ -6198,7 +6198,7 @@ TEST_P(LanceTimestampLtzInteTest, TestWriteAndReadInPinnedTimezone) { ASSERT_OK_AND_ASSIGN(std::vector> splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; @@ -6206,7 +6206,7 @@ TEST_P(LanceTimestampLtzInteTest, TestWriteAndReadInPinnedTimezone) { {"ltz_row", "ltz_added", "ltz_9", "id", "ltz_array", "ltz_0", "ltz_to_ntz", "ntz_to_ltz"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(splits)); @@ -6356,7 +6356,7 @@ TEST_P(TimeWriteAndReadInteTest, TestPKTableWithTimeColumns) { auto create_plan = [&](const std::shared_ptr& predicate) -> Result> { - TableLoadOptions scan_context_builder_load{table_path}; + TableLoadContext scan_context_builder_load{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -6386,7 +6386,7 @@ TEST_P(TimeWriteAndReadInteTest, TestPKTableWithTimeColumns) { const std::shared_ptr result_schema = arrow::schema(fields_with_row_kind); auto read = [&](const Plan& plan, const std::shared_ptr& predicate) -> Result> { - TableLoadOptions read_context_builder_load{table_path}; + TableLoadContext read_context_builder_load{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; @@ -6594,7 +6594,7 @@ TEST_P(TimeWriteAndReadInteTest, TestAppendTableWithTimePartitionAndValuePredica .ValueOrDie()); for (bool enable_scan_predicate : {false, true}) { SCOPED_TRACE(enable_scan_predicate ? "with scan predicate" : "without scan predicate"); - TableLoadOptions scan_context_builder_load{table_path}; + TableLoadContext scan_context_builder_load{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options = options; @@ -6619,7 +6619,7 @@ TEST_P(TimeWriteAndReadInteTest, TestAppendTableWithTimePartitionAndValuePredica : std::set({kEarlyPart, kLatePart}), planned_partitions); - TableLoadOptions read_context_builder_load{table_path}; + TableLoadContext read_context_builder_load{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index e29984fa3..e684eb8ad 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -35,7 +35,7 @@ #include "arrow/ipc/json_simple.h" #include "arrow/util/string_builder.h" #include "gtest/gtest.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/file_index/bitmap/bitmap_file_index_factory.h" @@ -143,7 +143,7 @@ class WriteAndReadInteWithIndexTest const std::vector> splits, const std::shared_ptr& predicate, const std::shared_ptr& expected_array) const { - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options["read.batch-size"] = "2"; @@ -154,7 +154,7 @@ class WriteAndReadInteWithIndexTest } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); @@ -1169,7 +1169,7 @@ class WriteAndReadInteWithIndexTest for (const auto& field : fields) { field_names.push_back(field->name()); } - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder_options["read.batch-size"] = "1"; @@ -1188,7 +1188,7 @@ class WriteAndReadInteWithIndexTest ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, - TableRead::Create(builder_load_options, std::move(context), builder_options)); + TableRead::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -1287,7 +1287,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestSimple) { &expected_array); ASSERT_TRUE(array_status.ok()); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options["read.batch-size"] = "2"; @@ -1301,7 +1301,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestSimple) { } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(batch_reader.get())); @@ -1366,7 +1366,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options[Options::FILE_FORMAT] = "orc"; @@ -1381,7 +1381,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(split)); // simulate read limits, only read 3 batches @@ -1656,7 +1656,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, literal_str.data(), literal_str.size())); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options["read.batch-size"] = "2"; @@ -1668,7 +1668,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) { } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(context_builder_load_options, std::move(read_context), + TableRead::Create(context_builder_load_context, std::move(read_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(std::vector>{split})); @@ -2346,7 +2346,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { auto predicate = PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, {Literal(10l), Literal(int64_t{2147483648})}); - TableLoadOptions builder_load_options{path}; + TableLoadContext builder_load_context{path}; std::map builder_options; ReadContextBuilder builder; builder_options["read.batch-size"] = "2"; @@ -2362,7 +2362,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr read, - TableRead::Create(builder_load_options, std::move(context), builder_options)); + TableRead::Create(builder_load_context, std::move(context), builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(split)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); @@ -3199,7 +3199,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - TableLoadOptions context_builder_load_options{path}; + TableLoadContext context_builder_load_context{path}; std::map context_builder_options; ReadContextBuilder context_builder; context_builder_options["read.batch-size"] = "2"; @@ -3210,7 +3210,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithIOException) { } ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); Result> table_read = TableRead::Create( - context_builder_load_options, std::move(read_context), context_builder_options); + context_builder_load_context, std::move(read_context), context_builder_options); CHECK_HOOK_STATUS(table_read.status(), i); Result> batch_reader = table_read.value()->CreateReader(split); CHECK_HOOK_STATUS(batch_reader.status(), i); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index d10319510..ae9461c1c 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -41,7 +41,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table_load_options.h" +#include "paimon/catalog/table_load_context.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_array.h" #include "paimon/common/data/binary_array_writer.h" @@ -290,14 +290,14 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface const std::vector>& data_splits, const std::string& blob_field, const std::vector>& expected_blobs) const { - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options = options; read_context_builder.SetReadFieldNames({blob_field}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_options, + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(data_splits)); @@ -355,14 +355,14 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface const std::vector>& commit_messages, bool ignore_empty_commit = true, int64_t commit_identifier = BATCH_WRITE_COMMIT_IDENTIFIER) const { - TableLoadOptions commit_builder_load_options{table_path}; + TableLoadContext commit_builder_load_context{table_path}; std::map commit_builder_options; CommitContextBuilder commit_builder("commit_user_1"); commit_builder.IgnoreEmptyCommit(ignore_empty_commit); PAIMON_ASSIGN_OR_RAISE(auto commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE( auto file_store_commit, - FileStoreCommit::Create(commit_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_builder_load_context, std::move(commit_context), commit_builder_options)); return file_store_commit->Commit(commit_messages, commit_identifier); } @@ -2025,14 +2025,14 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { ASSERT_EQ(3, batches_2.size()); // write data - TableLoadOptions context_builder_load_options{root_path}; + TableLoadContext context_builder_load_context{root_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.WithStreamingMode(true).Finish()); Result> write = FileStoreWrite::Create( - context_builder_load_options, std::move(write_context), context_builder_options); + context_builder_load_context, std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); // round 1 @@ -2464,7 +2464,7 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 200; i++) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; @@ -2472,7 +2472,7 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { context_builder.WithStreamingMode(true).Finish()); Result> write = FileStoreWrite::Create( - context_builder_load_options, std::move(write_context), context_builder_options); + context_builder_load_context, std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -2492,7 +2492,7 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { for (size_t i = 0; i < 400; i += paimon::test::RandomNumber(5, 15)) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - TableLoadOptions commit_context_builder_load_options{table_path}; + TableLoadContext commit_context_builder_load_context{table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -2500,7 +2500,7 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.IgnoreEmptyCommit(false).Finish()); auto commit = - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); @@ -2563,14 +2563,14 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) { /*partition_keys=*/{}, /*primary_keys=*/{}, options)); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); ::ArrowArray c_array; ASSERT_TRUE(arrow::ExportArray(*src_array, &c_array).ok()); @@ -2636,14 +2636,14 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { /*table_name=*/"append_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - std::string root_path = context_builder_load_options.path; + std::string root_path = context_builder_load_context.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); std::shared_ptr array = @@ -2676,24 +2676,24 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { ASSERT_TRUE(file_exist); // check read result - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::FILE_FORMAT] = file_format; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options[Options::FILE_FORMAT] = file_format; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2746,14 +2746,14 @@ TEST_P(WriteInteTest, TestPKTableWriteAndReadWithExternalPath) { /*table_name=*/"pk_table_with_external_path", &schema, /*partition_keys=*/{"f1"}, /*primary_keys=*/{"f0", "f1"}, options)); // write - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - std::string root_path = context_builder_load_options.path; + std::string root_path = context_builder_load_context.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); std::shared_ptr array = @@ -2969,14 +2969,14 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { /*primary_keys=*/{}, options)); // write - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - std::string root_path = context_builder_load_options.path; + std::string root_path = context_builder_load_context.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); auto write = [&](const std::string& json_array, const std::map& partition) -> void { @@ -3013,23 +3013,23 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { arrow::field("_VALUE_KIND", arrow::int8())); auto data_type = arrow::struct_(fields_with_row_kind); { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::FILE_FORMAT] = file_format; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options[Options::FILE_FORMAT] = file_format; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -3070,7 +3070,7 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::FILE_FORMAT] = file_format; @@ -3079,17 +3079,17 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { {{"dt", " "}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options[Options::FILE_FORMAT] = file_format; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -3107,7 +3107,7 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::FILE_FORMAT] = file_format; @@ -3116,17 +3116,17 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { {{"dt", ""}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options[Options::FILE_FORMAT] = file_format; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -3144,7 +3144,7 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } { - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::FILE_FORMAT] = file_format; @@ -3153,17 +3153,17 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { {{"dt", "__DEFAULT_PARTITION__"}, {"hr", "a=b?"}}}; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.SetPartitionFilter(partition_filters).Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options[Options::FILE_FORMAT] = file_format; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_options, + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); @@ -3204,14 +3204,14 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*table_name=*/"bar", &schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options)); // write - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); - std::string root_path = context_builder_load_options.path; + std::string root_path = context_builder_load_context.path; ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); std::shared_ptr array = @@ -3238,24 +3238,24 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { /*commit_identifier=*/1)); // check read result - TableLoadOptions scan_context_builder_load_options{table_path}; + TableLoadContext scan_context_builder_load_context{table_path}; std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; scan_context_builder_options[Options::FILE_FORMAT] = file_format; ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, - TableScan::Create(scan_context_builder_load_options, + TableScan::Create(scan_context_builder_load_context, std::move(scan_context), scan_context_builder_options)); - TableLoadOptions read_context_builder_load_options{table_path}; + TableLoadContext read_context_builder_load_context{table_path}; std::map read_context_builder_options; ReadContextBuilder read_context_builder; read_context_builder_options[Options::FILE_FORMAT] = file_format; ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, - TableRead::Create(read_context_builder_load_options, + TableRead::Create(read_context_builder_load_context, std::move(read_context), read_context_builder_options)); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -3313,7 +3313,7 @@ TEST_P(WriteInteTest, TestWriteWithIOException) { ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; @@ -3321,7 +3321,7 @@ TEST_P(WriteInteTest, TestWriteWithIOException) { context_builder.WithStreamingMode(true).Finish()); Result> write = FileStoreWrite::Create( - context_builder_load_options, std::move(write_context), context_builder_options); + context_builder_load_context, std::move(write_context), context_builder_options); CHECK_HOOK_STATUS(write.status(), i); auto& file_store_write = write.value(); @@ -3369,14 +3369,14 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, options)); // write data - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -3393,7 +3393,7 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { std::string tmp_table_path = tmp_dir->Str(); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); - TableLoadOptions commit_context_builder_load_options{tmp_table_path}; + TableLoadContext commit_context_builder_load_context{tmp_table_path}; std::map commit_context_builder_options; CommitContextBuilder commit_context_builder("commit_user_1"); commit_context_builder_options[Options::MANIFEST_TARGET_FILE_SIZE] = "8mb"; @@ -3401,7 +3401,7 @@ TEST_P(WriteInteTest, TestCommitWithIOException) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.IgnoreEmptyCommit(false).Finish()); auto commit = - FileStoreCommit::Create(commit_context_builder_load_options, std::move(commit_context), + FileStoreCommit::Create(commit_context_builder_load_context, std::move(commit_context), commit_context_builder_options); CHECK_HOOK_STATUS(commit.status(), i); Result committed = commit.value()->FilterAndCommit({{0, commit_messages}}); @@ -3450,7 +3450,7 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) { /*primary_keys=*/{}, options)); std::shared_ptr write_pool = GetMemoryPool(); { - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; @@ -3459,7 +3459,7 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) { context_builder.WithMemoryPool(write_pool).WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_0, MakeRecordBatch({{"Alice", 10, 1, 11.1}}, {{"f1", "10"}}, 0)); @@ -3637,14 +3637,14 @@ TEST_F(WriteInteTest, TestBranchWrite) { std::map options = {{Options::FILE_FORMAT, "orc"}, {Options::FILE_SYSTEM, "local"}}; - TableLoadOptions context_builder_load_options{table_path}; + TableLoadContext context_builder_load_context{table_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; context_builder_options["branch"] = "rt"; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, - FileStoreWrite::Create(context_builder_load_options, + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); // write to rt branch, pk table @@ -3686,14 +3686,14 @@ TEST_F(WriteInteTest, TestBranchWrite) { ASSERT_TRUE(expected_commit_message_0->TEST_Equal(*result_commit_msgs)); // write to main branch, append table - TableLoadOptions context_builder1_load_options{table_path}; + TableLoadContext context_builder1_load_context{table_path}; std::map context_builder1_options; WriteContextBuilder context_builder1("commit_user_1"); context_builder1_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context1, context_builder1.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr file_store_write1, - FileStoreWrite::Create(context_builder1_load_options, std::move(write_context1), + FileStoreWrite::Create(context_builder1_load_context, std::move(write_context1), context_builder1_options)); std::string data_1 = R"([["20240725", "watermelon", 3], @@ -3828,13 +3828,13 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { auto write_array = [&](const std::vector& write_cols, const std::shared_ptr& write_array) -> Result>> { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; WriteContextBuilder builder(commit_user); builder.WithWriteSchema(write_cols); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_write, - FileStoreWrite::Create(builder_load_options, + FileStoreWrite::Create(builder_load_context, std::move(write_context), builder_options)); ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*write_array, &c_array)); @@ -3865,12 +3865,12 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { auto commit = [&](const std::vector>& commit_msgs, int64_t latest_snapshot_id, int64_t next_row_id) { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; CommitContextBuilder builder(commit_user); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_commit, - FileStoreCommit::Create(builder_load_options, + FileStoreCommit::Create(builder_load_context, std::move(commit_context), builder_options)); ASSERT_OK(file_store_commit->Commit(commit_msgs)); @@ -3884,13 +3884,13 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) { // first_row_id, write_cols, min_sequence_number, max_sequence_number using MetaType = std::tuple>, int64_t, int64_t>; auto check_committed_meta = [&](const std::vector& expected_meta) { - TableLoadOptions builder_load_options{table_path}; + TableLoadContext builder_load_context{table_path}; std::map builder_options; ScanContextBuilder builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr table_scan, - TableScan::Create(builder_load_options, std::move(scan_context), builder_options)); + TableScan::Create(builder_load_context, std::move(scan_context), builder_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); auto data_split = std::dynamic_pointer_cast(plan->Splits()[0]); @@ -4266,14 +4266,14 @@ TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) { /*primary_keys=*/{"id"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder_options = options; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); ASSERT_OK_AND_ASSIGN( @@ -4334,13 +4334,13 @@ TEST_P(WriteInteTest, TestPkSpillableDiskQuotaExhaustedFallsBackToFlush) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); auto write_array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ @@ -4394,13 +4394,13 @@ TEST_P(WriteInteTest, TestPkSpillableGlobalMemoryPreemptionDataCorrectness) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); // Write long strings to different partitions to trigger global memory preemption. @@ -4496,13 +4496,13 @@ TEST_P(WriteInteTest, TestPkSpillableTempFilesCleanedAfterPrepareCommit) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); auto batch1 = @@ -4564,13 +4564,13 @@ TEST_P(WriteInteTest, TestPkSpillableIntermediateMergeWithTempFileTracking) { std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); auto write_array_fn = [](FileStoreWrite* writer, @@ -4651,13 +4651,13 @@ TEST_P(WriteInteTest, TestPkSpillableVector) { /*primary_keys=*/{"pt", "f0"}, options)); std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); auto write_array = [](FileStoreWrite* writer, const std::shared_ptr& array) { @@ -4738,7 +4738,7 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { return writer->Write(std::move(batch)); }; - TableLoadOptions write_builder_load_options{table_path}; + TableLoadContext write_builder_load_context{table_path}; std::map write_builder_options; WriteContextBuilder write_builder("commit_user_1"); write_builder.WithStreamingMode(true) @@ -4746,7 +4746,7 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { .SetWriteBufferSpillThreadNumber(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, - FileStoreWrite::Create(write_builder_load_options, + FileStoreWrite::Create(write_builder_load_context, std::move(write_context), write_builder_options)); // Round 1: Bucket 0 writes Alice(dup) + Bob, Bucket 1 writes Dave(dup) + Eve @@ -4935,7 +4935,7 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { // Write with spill enabled (WithTempDirectory) std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); - TableLoadOptions context_builder_load_options{root_path}; + TableLoadContext context_builder_load_context{root_path}; std::map context_builder_options; WriteContextBuilder context_builder("commit_user_1"); context_builder_options = options; @@ -4944,7 +4944,7 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { context_builder.WithStreamingMode(true).WithTempDirectory(tmp_dir).Finish()); ASSERT_OK_AND_ASSIGN( auto file_store_write, - FileStoreWrite::Create(context_builder_load_options, std::move(write_context), + FileStoreWrite::Create(context_builder_load_context, std::move(write_context), context_builder_options)); // Start IOHook only for spill write/read operations. From dee1d3d080280c854f2ba57e47c0940384e6c0b7 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Sat, 14 Nov 2026 06:47:19 +0800 Subject: [PATCH 5/6] refactor(table): make copy semantics and factory options explicit --- docs/source/user_guide/catalog.rst | 18 +- docs/source/user_guide/global_index.rst | 4 +- docs/source/user_guide/read.rst | 2 +- include/paimon/catalog/table.h | 3 +- include/paimon/file_store_commit.h | 2 +- include/paimon/file_store_write.h | 2 +- include/paimon/table/format/format_table.h | 2 +- include/paimon/table/source/table_read.h | 2 +- include/paimon/table/source/table_scan.h | 2 +- .../core/operation/file_store_write_test.cpp | 2 +- src/paimon/core/table/file_store_table.cpp | 2 +- src/paimon/core/table/file_store_table.h | 2 +- src/paimon/core/table/format/format_table.cpp | 2 +- .../core/table/format/format_table_test.cpp | 40 +++-- .../core/table/source/table_read_test.cpp | 25 +-- .../table/system/audit_log_system_table.cpp | 7 +- .../table/system/audit_log_system_table.h | 2 +- .../core/table/system/binlog_system_table.cpp | 2 +- .../core/table/system/binlog_system_table.h | 2 +- .../table/system/global_system_tables.cpp | 8 +- .../core/table/system/global_system_tables.h | 8 +- .../table/system/metadata_system_tables.cpp | 16 +- .../table/system/metadata_system_tables.h | 16 +- .../system/read_optimized_system_table.cpp | 7 +- .../system/read_optimized_system_table.h | 2 +- src/paimon/core/table/table_factory.cpp | 8 +- src/paimon/core/table/table_test.cpp | 162 ++++++++++-------- src/paimon/rest/rest_catalog_test.cpp | 6 +- test/inte/read_inte_test.cpp | 56 +++--- 29 files changed, 220 insertions(+), 192 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index f40652c69..c6ddba5d8 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -148,7 +148,7 @@ its data-format plugin; the plugin is resolved when creating an operator that uses the format. ``NewCommit()``, ``NewWrite()``, ``NewRead()`` and ``NewScan()`` take an operation -Context containing execution settings. Apply dynamic options with ``Table::Copy()`` +Context containing execution settings. Apply dynamic options with ``Table::CopyWithoutTimeTravel()`` before creating operators; the original table remains unchanged. Operators use the copy's bound type, path, schema, options, file system and catalog services. ``GetSchema()`` returns the @@ -157,9 +157,13 @@ Branch or type options that contradict a loaded managed table are rejected. See the :ref:`Table API `. The static factories instead take ``TableLoadContext``, an operation Context and -an optional dynamic options map. ``TableLoadContext`` holds the table identity, +an explicit dynamic options map (pass ``{}`` when no overrides are needed). +``TableLoadContext`` holds the table identity, storage binding and supplied metadata. Operation Contexts hold execution settings; -finishing an operation Context performs no table metadata I/O: +finishing an operation Context performs no table metadata I/O. +``CopyWithoutTimeTravel`` preserves the bound table schema, matching Java's +``copyWithoutTimeTravel``. It does not resolve a historical schema from snapshot +or timestamp options: .. code-block:: cpp @@ -171,7 +175,7 @@ finishing an operation Context performs no table metadata I/O: PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, paimon::CommitContextBuilder("commit-user").Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - paimon::FileStoreCommit::Create(load_context, std::move(context))); + paimon::FileStoreCommit::Create(load_context, std::move(context), {})); For file-system loading, set ``TableLoadContext::path``. Optional fields specify an explicit file system, URI scheme routing or a serialized Paimon table schema. @@ -243,7 +247,7 @@ keep it across recovery. PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, paimon::CommitContextBuilder("writer-1").Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr commit_table, - table->Copy( + table->CopyWithoutTimeTravel( {{paimon::Options::REST_COMMIT_ACQUIRE_LOCK, "true"}})); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, commit_table->NewCommit(std::move(context))); @@ -560,7 +564,7 @@ These examples use ``shared_catalog`` from `Reading through the catalog`_. .SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - TableScan::Create(scan_load, std::move(scan_context))); + TableScan::Create(scan_load, std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); // A worker without the catalog tells the read that the table enables query auth. @@ -592,7 +596,7 @@ hands out the credentials of that branch. PAIMON_ASSIGN_OR_RAISE(std::unique_ptr dev_scan_context, dev_scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr dev_scan, - TableScan::Create(dev_load, std::move(dev_scan_context))); + TableScan::Create(dev_load, std::move(dev_scan_context), {})); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr dev_plan, dev_scan->CreatePlan()); // The worker reads the same branch. diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 3d9663879..d24453ebb 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -324,7 +324,7 @@ them. A search whose ``full-text-index.search-mode``, or else ``global-index.sea PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(load_context, std::move(scan_context))); + paimon::TableScan::Create(load_context, std::move(scan_context), {})); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_plan, scan->CreatePlan()); paimon::ReadContextBuilder read_builder; @@ -332,7 +332,7 @@ them. A search whose ``full-text-index.search-mode``, or else ``global-index.sea PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - paimon::TableRead::Create(load_context, std::move(read_context))); + paimon::TableRead::Create(load_context, std::move(read_context), {})); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(data_plan->Splits())); diff --git a/docs/source/user_guide/read.rst b/docs/source/user_guide/read.rst index 5566d2981..91e4f99f9 100644 --- a/docs/source/user_guide/read.rst +++ b/docs/source/user_guide/read.rst @@ -53,7 +53,7 @@ In a function returning ``Status`` or ``Result``, use the ordinary scan API: ScanContextBuilder builder; builder.WithMemoryPool(pool); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, TableScan::Create(load_context, std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, TableScan::Create(load_context, std::move(context), {})); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); * The pool controls only allocations routed through it, not all planning memory diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 2b43a66d7..b1cf9d191 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -87,7 +87,8 @@ class PAIMON_EXPORT Table { /// bindings are retained; load another table to select a different branch. /// Managed tables reject changes to immutable schema options (such as bucket-key), and /// validate conditional option changes using the original table's configuration. - virtual Result> Copy( + /// Keeps the bound schema; snapshot/time options do not load a historical table schema. + virtual Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const = 0; /// Creates a scan using this table's schema and options. diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index 2af9209ba..26044882d 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -60,7 +60,7 @@ class PAIMON_EXPORT FileStoreCommit { /// @return A Result containing a unique pointer to the `FileStoreCommit` instance. static Result> Create( const TableLoadContext& load_context, std::unique_ptr context, - const std::map& dynamic_options = {}); + const std::map& dynamic_options); virtual ~FileStoreCommit() = default; diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 686e5135a..6761e20dd 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -56,7 +56,7 @@ class PAIMON_EXPORT FileStoreWrite { /// @return A Result containing a unique pointer to the `FileStoreWrite` instance. static Result> Create( const TableLoadContext& load_context, std::unique_ptr context, - const std::map& dynamic_options = {}); + const std::map& dynamic_options); virtual ~FileStoreWrite() = default; diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index da6be3971..b1892f351 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -117,7 +117,7 @@ class PAIMON_EXPORT FormatTable : public Table { ~FormatTable() override; - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; Result> NewScan(std::unique_ptr context) const override; diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index b093aa11b..613198d87 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -51,7 +51,7 @@ class PAIMON_EXPORT TableRead { /// @return A Result containing a unique pointer to the `TableRead` instance. static Result> Create( const TableLoadContext& load_context, std::unique_ptr context, - const std::map& dynamic_options = {}); + const std::map& dynamic_options); /// Creates a `BatchReader` instance for reading data. /// diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index aa2e4fe42..67850d612 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -45,7 +45,7 @@ class PAIMON_EXPORT TableScan { /// @return A Result containing a unique pointer to the `TableScan` instance. static Result> Create( const TableLoadContext& load_context, std::unique_ptr context, - const std::map& dynamic_options = {}); + const std::map& dynamic_options); virtual ~TableScan() = default; diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 2a2419b74..7efdaa796 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -300,7 +300,7 @@ TEST(FileStoreWriteTest, TestCreateWithInvalidInput) { ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx2, builder.WithExecutor(nullptr).Finish()); ASSERT_NOK(FileStoreWrite::Create(builder_load_context, std::move(ctx2), builder_options)); - ASSERT_NOK(FileStoreWrite::Create({}, /*context=*/nullptr)); + ASSERT_NOK(FileStoreWrite::Create({}, /*context=*/nullptr, {})); } TEST(FileStoreWriteTest, TestCreateAppendTable) { diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 42e253a27..94d995b7a 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -316,7 +316,7 @@ Result FileStoreTable::ResolveOptions( return CoreOptions::FromMap(options, file_system_, fs_scheme_map_); } -Result> FileStoreTable::Copy( +Result> FileStoreTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { using OptionsMap = std::map; PAIMON_ASSIGN_OR_RAISE(OptionsMap options, ResolveOptionMap(dynamic_options)); diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 97f0168a6..a3dfdf7bb 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -97,7 +97,7 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; Result> NewScan(std::unique_ptr context) const override; diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 0bd1f033c..b89c3abec 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -320,7 +320,7 @@ Result> NewFormatTableCommit( return std::unique_ptr(std::move(format_commit)); } } // namespace -Result> FormatTable::Copy( +Result> FormatTable::CopyWithoutTimeTravel( const std::map& options) const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, Copy(std::make_shared(*this), options)); diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 307ed2187..a18242373 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -673,7 +673,7 @@ TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); auto read = entrypoint != 1 ? ([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - loaded_table->Copy(context_options)); + loaded_table->CopyWithoutTimeTravel(context_options)); return configured->NewRead(std::move(context)); })() : TableRead::Create(context_load_context, @@ -688,7 +688,7 @@ TEST(FormatTableTest, TestReadRejectsSearchForDirectAndStaticEntrypoints) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder.Finish()); ASSERT_OK(([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - loaded_table->Copy(context_builder_options)); + loaded_table->CopyWithoutTimeTravel(context_builder_options)); return configured->NewRead(std::move(context)); })()); } @@ -1192,7 +1192,7 @@ TEST(FormatTableTest, TestCatalogFactoriesRetainExternalLocationWithExplicitPath ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, WriteContextBuilder("writer").Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, - FileStoreWrite::Create(load, std::move(write_context))); + FileStoreWrite::Create(load, std::move(write_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({++id}, {partition}, partition, {{"dt", partition}})); ASSERT_OK(writer->Write(std::move(batch))); @@ -1201,7 +1201,7 @@ TEST(FormatTableTest, TestCatalogFactoriesRetainExternalLocationWithExplicitPath ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, CommitContextBuilder("writer").Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(load, std::move(commit_context))); + FileStoreCommit::Create(load, std::move(commit_context), {})); ASSERT_OK(commit->Commit(messages)); ASSERT_OK(writer->Close()); } @@ -1220,13 +1220,13 @@ TEST(FormatTableTest, TestCatalogFactoriesRetainExternalLocationWithExplicitPath ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(load, std::move(scan_context))); + TableScan::Create(load, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 2u); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(load, std::move(read_context))); + TableRead::Create(load, std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); std::vector partitions; @@ -1248,7 +1248,7 @@ TEST(FormatTableTest, TestCatalogFactoriesRetainExternalLocationWithExplicitPath load.path = dir->Str() + "/other"; ASSERT_OK_AND_ASSIGN(std::unique_ptr mismatch_context, ScanContextBuilder().Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(load, std::move(mismatch_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(load, std::move(mismatch_context), {}), "does not match the catalog table location"); } } @@ -3050,7 +3050,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_3, - write_builder_table->Copy(write_builder_options)); + write_builder_table->CopyWithoutTimeTravel(write_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, configured_table_3->NewWrite(std::move(write_context))); for (int32_t i = 1; i <= 3; i++) { @@ -3067,7 +3067,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { CommitContextBuilder commit_builder(""); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_4, - commit_builder_table->Copy(commit_builder_options)); + commit_builder_table->CopyWithoutTimeTravel(commit_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, configured_table_4->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); @@ -3078,7 +3078,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ScanContextBuilder scan_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_5, - scan_builder_table->Copy(scan_builder_options)); + scan_builder_table->CopyWithoutTimeTravel(scan_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, configured_table_5->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); @@ -3090,7 +3090,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ReadContextBuilder read_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_6, - read_builder_table->Copy(read_builder_options)); + read_builder_table->CopyWithoutTimeTravel(read_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, configured_table_6->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); @@ -3135,8 +3135,9 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - scan_builder_table->Copy(scan_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + scan_builder_table->CopyWithoutTimeTravel(scan_builder_options)); return configured->NewScan(std::move(scan_context)); })(), "has no branches"); @@ -3148,8 +3149,9 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - read_builder_table->Copy(read_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + read_builder_table->CopyWithoutTimeTravel(read_builder_options)); return configured->NewRead(std::move(read_context)); })(), "has no branches"); @@ -3163,7 +3165,7 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) ASSERT_NOK_WITH_MSG(([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr
configured, - write_builder_table->Copy(write_builder_options)); + write_builder_table->CopyWithoutTimeTravel(write_builder_options)); return configured->NewWrite(std::move(write_context)); })(), "has no branches"); @@ -3177,9 +3179,9 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr
configured, - commit_builder_table->Copy(commit_builder_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + commit_builder_table->CopyWithoutTimeTravel( + commit_builder_options)); return configured->NewCommit(std::move(commit_context)); })(), "has no branches"); diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 079e1c72b..ec6f31e36 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -179,7 +179,8 @@ TEST(TableReadTest, TestFullTextMatchReadEntryAndScoreValidation) { auto context_options = builder_options; ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); ASSERT_OK(table_entry ? ([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, table->Copy(context_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->CopyWithoutTimeTravel(context_options)); return configured->NewRead(std::move(context)); })() : TableRead::Create(context_load_context, std::move(context), @@ -201,14 +202,15 @@ TEST(TableReadTest, TestFullTextMatchReadEntryAndScoreValidation) { } context_options = builder_options; ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - ASSERT_NOK_WITH_MSG(table_entry ? ([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(context_options)); - return configured->NewRead(std::move(context)); - })() - : TableRead::Create(context_load_context, - std::move(context), context_options), - "Score-producing FullTextMatch cannot appear under OR"); + ASSERT_NOK_WITH_MSG( + table_entry + ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->CopyWithoutTimeTravel(context_options)); + return configured->NewRead(std::move(context)); + })() + : TableRead::Create(context_load_context, std::move(context), context_options), + "Score-producing FullTextMatch cannot appear under OR"); } } @@ -231,7 +233,8 @@ TEST(TableReadTest, TestPerFieldScoreReadEntry) { {"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); ASSERT_OK(table_entry ? ([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, table->Copy(builder_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->CopyWithoutTimeTravel(builder_options)); return configured->NewRead(std::move(context)); })() : TableRead::Create(builder_load_context, std::move(context), @@ -391,7 +394,7 @@ TEST(TableReadTest, TestPreciseEvaluationRejectsSystemAndFormatTables) { ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); ASSERT_NOK_WITH_MSG(loaded_table ? format_table->NewRead(std::move(context)) - : TableRead::Create(builder_load_context, std::move(context)), + : TableRead::Create(builder_load_context, std::move(context), {}), "Precise file-index evaluation is not supported for format tables"); } } diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index c408e1623..818800883 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -423,7 +423,7 @@ Result> AuditLogSystemTable::NewScan( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, CreateDataTable(table_schema_, options_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, - data_table->Copy(dynamic_options)); + data_table->CopyWithoutTimeTravel(dynamic_options)); return configured_table->NewScan(std::move(data_context)); } @@ -455,7 +455,8 @@ Result> AuditLogSystemTable::NewChangelogRead( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, CreateDataTable(table_schema_, options_)); auto options = read_options; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, data_table->Copy(options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, + data_table->CopyWithoutTimeTravel(options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read, configured_table->NewRead(std::move(data_context))); auto* key_value_read = dynamic_cast(data_read.get()); @@ -519,7 +520,7 @@ Result> AuditLogSystemTable::ReadOptions( return read_options; } -Result> AuditLogSystemTable::Copy( +Result> AuditLogSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { using OptionsMap = std::map; PAIMON_ASSIGN_OR_RAISE(OptionsMap options, diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 192188b62..afed35bc1 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -52,7 +52,7 @@ std::unique_ptr CreateChangelogBatchReader( /// System table for `T$audit_log`, exposing row-level changelog records with rowkind. class AuditLogSystemTable : public SystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "audit_log"; diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 2348b4fc9..80ed9dfd2 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -104,7 +104,7 @@ Result> BinlogSystemTable::NewRead( return NewChangelogRead(context, CreateBinlogBatchConverter(), dynamic_options); } -Result> BinlogSystemTable::Copy( +Result> BinlogSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { using OptionsMap = std::map; PAIMON_ASSIGN_OR_RAISE(OptionsMap options, diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index eb7722054..d0d47ac6c 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -34,7 +34,7 @@ std::shared_ptr CreateBinlogBatchConverter(); /// System table for `T$binlog`, exposing changelog records with list-wrapped data columns. class BinlogSystemTable : public AuditLogSystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "binlog"; diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 06a4d8994..096bbbca0 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -598,25 +598,25 @@ Result> PartitionsSystemTable::BuildRows() const { return rows; } -Result> CatalogOptionsSystemTable::Copy( +Result> CatalogOptionsSystemTable::CopyWithoutTimeTravel( const std::map& /*dynamic_options*/) const { auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); } -Result> AllTableOptionsSystemTable::Copy( +Result> AllTableOptionsSystemTable::CopyWithoutTimeTravel( const std::map& /*dynamic_options*/) const { auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); } -Result> TablesSystemTable::Copy( +Result> TablesSystemTable::CopyWithoutTimeTravel( const std::map& /*dynamic_options*/) const { auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); } -Result> PartitionsSystemTable::Copy( +Result> PartitionsSystemTable::CopyWithoutTimeTravel( const std::map& /*dynamic_options*/) const { auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index e1fc1be95..b69b29bbb 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -46,7 +46,7 @@ struct GlobalSystemTableContext { /// as key/value rows. class CatalogOptionsSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "catalog_options"; @@ -65,7 +65,7 @@ class CatalogOptionsSystemTable : public InMemorySystemTable { /// all databases as (database_name, table_name, key, value) rows. class AllTableOptionsSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "all_table_options"; @@ -83,7 +83,7 @@ class AllTableOptionsSystemTable : public InMemorySystemTable { /// databases including record counts and file statistics. class TablesSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "tables"; @@ -101,7 +101,7 @@ class TablesSystemTable : public InMemorySystemTable { /// for all tables across all databases. class PartitionsSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "partitions"; diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 52b74ba62..17e726b21 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -1004,49 +1004,49 @@ Result> FilesSystemTable::BuildRows() const { return rows; } -Result> OptionsSystemTable::Copy( +Result> OptionsSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); } -Result> SnapshotsSystemTable::Copy( +Result> SnapshotsSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); } -Result> SchemasSystemTable::Copy( +Result> SchemasSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); } -Result> TagsSystemTable::Copy( +Result> TagsSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); } -Result> BranchesSystemTable::Copy( +Result> BranchesSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); } -Result> ConsumersSystemTable::Copy( +Result> ConsumersSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); auto table = std::make_shared(*this); return std::shared_ptr
(std::move(table)); } -Result> ManifestsSystemTable::Copy( +Result> ManifestsSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); PAIMON_ASSIGN_OR_RAISE(MetadataSystemTableContext context, @@ -1056,7 +1056,7 @@ Result> ManifestsSystemTable::Copy( return std::shared_ptr
(std::move(table)); } -Result> FilesSystemTable::Copy( +Result> FilesSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { PAIMON_RETURN_NOT_OK(CheckBranch(dynamic_options)); PAIMON_ASSIGN_OR_RAISE(MetadataSystemTableContext context, diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index a44c3ff72..aa7bf6f4b 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -33,7 +33,7 @@ class TableSchema; /// System table for `T$options`, exposing the latest base table options as key/value rows. class OptionsSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "options"; @@ -60,7 +60,7 @@ struct MetadataSystemTableContext { /// System table for `T$snapshots`, exposing snapshot commit history. class SnapshotsSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "snapshots"; @@ -78,7 +78,7 @@ class SnapshotsSystemTable : public InMemorySystemTable { /// System table for `T$schemas`, exposing schema evolution history. class SchemasSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "schemas"; @@ -95,7 +95,7 @@ class SchemasSystemTable : public InMemorySystemTable { /// System table for `T$tags`, exposing tags and the snapshots they reference. class TagsSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "tags"; @@ -112,7 +112,7 @@ class TagsSystemTable : public InMemorySystemTable { /// System table for `T$branches`, exposing table branches including `main`. class BranchesSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "branches"; @@ -129,7 +129,7 @@ class BranchesSystemTable : public InMemorySystemTable { /// System table for `T$consumers`, exposing persisted streaming consumer offsets. class ConsumersSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "consumers"; @@ -147,7 +147,7 @@ class ConsumersSystemTable : public InMemorySystemTable { /// System table for `T$manifests`, exposing data manifest metadata in the latest snapshot. class ManifestsSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "manifests"; @@ -169,7 +169,7 @@ class ManifestsSystemTable : public InMemorySystemTable { /// System table for `T$files`, exposing data file metadata in the latest snapshot. class FilesSystemTable : public InMemorySystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "files"; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index a48c13ed6..9deb54366 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -90,7 +90,8 @@ Result> ReadOptimizedSystemTable::NewScan( CreateDataTable(table_schema_, options)); auto scan_options = dynamic_options; scan_options[kReadOptimizedScanOption] = "true"; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, data_table->Copy(scan_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, + data_table->CopyWithoutTimeTravel(scan_options)); return configured_table->NewScan(std::move(base_context)); } @@ -133,11 +134,11 @@ Result> ReadOptimizedSystemTable::NewRead( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_table, CreateDataTable(table_schema_, options_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table, - data_table->Copy(dynamic_options)); + data_table->CopyWithoutTimeTravel(dynamic_options)); return configured_table->NewRead(std::move(base_context)); } -Result> ReadOptimizedSystemTable::Copy( +Result> ReadOptimizedSystemTable::CopyWithoutTimeTravel( const std::map& dynamic_options) const { using OptionsMap = std::map; PAIMON_ASSIGN_OR_RAISE(OptionsMap options, diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 5513585ab..c9cc7e36c 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -32,7 +32,7 @@ class TableSchema; /// System table for `T$ro`, exposing read-optimized data. class ReadOptimizedSystemTable : public SystemTable { public: - Result> Copy( + Result> CopyWithoutTimeTravel( const std::map& dynamic_options) const override; static constexpr const char* kName = "ro"; diff --git a/src/paimon/core/table/table_factory.cpp b/src/paimon/core/table/table_factory.cpp index 417dd0480..e0446a3f3 100644 --- a/src/paimon/core/table/table_factory.cpp +++ b/src/paimon/core/table/table_factory.cpp @@ -277,7 +277,7 @@ Result> TableRead::Create( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(load_context, dynamic_options, "read")); - PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(table, table->CopyWithoutTimeTravel(dynamic_options)); return table->NewRead(std::move(context)); } @@ -296,7 +296,7 @@ Result> TableScan::Create( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(load_context, dynamic_options, "scan")); - PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(table, table->CopyWithoutTimeTravel(dynamic_options)); return table->NewScan(std::move(context)); } @@ -315,7 +315,7 @@ Result> FileStoreWrite::Create( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(load_context, dynamic_options, "write")); - PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(table, table->CopyWithoutTimeTravel(dynamic_options)); return table->NewWrite(std::move(context)); } @@ -332,7 +332,7 @@ Result> FileStoreCommit::Create( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableFactory::Create(load_context, dynamic_options, "commit")); - PAIMON_ASSIGN_OR_RAISE(table, table->Copy(dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(table, table->CopyWithoutTimeTravel(dynamic_options)); return table->NewCommit(std::move(context)); } diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 2a569b5f3..957a4fbc3 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -140,11 +140,11 @@ TEST(TableTest, TestCopyRetainsBindingsAndMergesOptionsIndependently) { FileStoreTable::Create(fs, path, checked_pointer_cast(created->GetSchema()), Identifier("db", "tbl"), "catalog-uuid", "main", routing, catalog)); const auto original_options = checked_pointer_cast(table->GetSchema())->Options(); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
first, - table->Copy({{Options::TARGET_FILE_ROW_NUM, "1"}, {Options::FILE_FORMAT, "parquet"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
first, + table->CopyWithoutTimeTravel({{Options::TARGET_FILE_ROW_NUM, "1"}, + {Options::FILE_FORMAT, "parquet"}})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
second, - first->Copy({{Options::TARGET_FILE_ROW_NUM, "2"}})); + first->CopyWithoutTimeTravel({{Options::TARGET_FILE_ROW_NUM, "2"}})); ASSERT_EQ(original_options, checked_pointer_cast(table->GetSchema())->Options()); ASSERT_EQ("1", checked_pointer_cast(first->GetSchema()) ->Options() @@ -161,8 +161,10 @@ TEST(TableTest, TestCopyRetainsBindingsAndMergesOptionsIndependently) { ASSERT_EQ(fs, second->GetFileSystem()); ASSERT_EQ(routing, checked_pointer_cast(second)->GetFileSystemSchemeToIdentifierMap()); - ASSERT_NOK_WITH_MSG(second->Copy({{Options::BRANCH, "dev"}}), "branch contradicts"); - ASSERT_NOK_WITH_MSG(second->Copy({{Options::TYPE, "format-table"}}), "type contradicts"); + ASSERT_NOK_WITH_MSG(second->CopyWithoutTimeTravel({{Options::BRANCH, "dev"}}), + "branch contradicts"); + ASSERT_NOK_WITH_MSG(second->CopyWithoutTimeTravel({{Options::TYPE, "format-table"}}), + "type contradicts"); catalog.reset(); table.reset(); first.reset(); @@ -203,11 +205,12 @@ TEST(TableTest, TestCopyRejectsBucketKeyChangesWithoutLosingRows) { ASSERT_OK_AND_ASSIGN(std::vector> batches, generator.SplitArrayByPartitionAndBucket({row})); ASSERT_OK(helper->WriteAndCommit(std::move(batches), 0, std::nullopt)); - ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "b"}}), "immutable option 'bucket-key'"); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
copied, - table->Copy({{Options::BUCKET_KEY, "a"}, {Options::READ_BATCH_SIZE, "1"}})); - ASSERT_NOK_WITH_MSG(copied->Copy({{Options::BUCKET_KEY, "b"}}), + ASSERT_NOK_WITH_MSG(table->CopyWithoutTimeTravel({{Options::BUCKET_KEY, "b"}}), + "immutable option 'bucket-key'"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->CopyWithoutTimeTravel( + {{Options::BUCKET_KEY, "a"}, {Options::READ_BATCH_SIZE, "1"}})); + ASSERT_NOK_WITH_MSG(copied->CopyWithoutTimeTravel({{Options::BUCKET_KEY, "b"}}), "immutable option 'bucket-key'"); ASSERT_EQ(schema.value()->Options(), original_options); auto predicate = PredicateBuilder::Equal(1, "b", FieldType::INT, Literal(int32_t{1})); @@ -247,11 +250,11 @@ TEST(TableTest, TestCopyNormalizesSchemaKeysAndRetainsPath) { FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl"), "uuid", "main")); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, - table->Copy({{"primary-key", " a, , b "}, - {"partition", "a, "}, - {"type", "TABLE"}, - {"path", "another/table"}, - {Options::READ_BATCH_SIZE, "32"}})); + table->CopyWithoutTimeTravel({{"primary-key", " a, , b "}, + {"partition", "a, "}, + {"type", "TABLE"}, + {"path", "another/table"}, + {Options::READ_BATCH_SIZE, "32"}})); auto copied_schema = checked_pointer_cast(copied->GetSchema()); ASSERT_EQ(copied_schema->Id(), 7); ASSERT_EQ(copied_schema->Options().count("primary-key"), 0u); @@ -259,8 +262,8 @@ TEST(TableTest, TestCopyNormalizesSchemaKeysAndRetainsPath) { ASSERT_EQ(copied_schema->Options().count("type"), 0u); ASSERT_EQ(copied_schema->Options().at("path"), table->GetRootPath()); ASSERT_EQ(copied_schema->Options().at(Options::READ_BATCH_SIZE), "32"); - ASSERT_NOK_WITH_MSG(copied->Copy({{"primary-key", "b,a"}}), "primary-key"); - ASSERT_NOK_WITH_MSG(copied->Copy({{"partition", "b"}}), "partition"); + ASSERT_NOK_WITH_MSG(copied->CopyWithoutTimeTravel({{"primary-key", "b,a"}}), "primary-key"); + ASSERT_NOK_WITH_MSG(copied->CopyWithoutTimeTravel({{"partition", "b"}}), "partition"); ASSERT_EQ(schema->Options().count(Options::READ_BATCH_SIZE), 0u); } @@ -277,11 +280,13 @@ TEST(TableTest, TestSystemTableCopiesPreserveManagedOptionRules) { SCOPED_TRACE(kind); TableLoadContext load{dir->Str() + "$" + kind}; ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, TableFactory::Create(load, {}, "read")); - ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "b"}}), "bucket-key"); + ASSERT_NOK_WITH_MSG(table->CopyWithoutTimeTravel({{Options::BUCKET_KEY, "b"}}), + "bucket-key"); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, - table->Copy({{Options::IGNORE_DELETE, "true"}})); - ASSERT_NOK_WITH_MSG(copied->Copy({{Options::IGNORE_DELETE, "false"}}), "ignore-delete"); - ASSERT_OK(table->Copy({{Options::IGNORE_DELETE, "false"}})); + table->CopyWithoutTimeTravel({{Options::IGNORE_DELETE, "true"}})); + ASSERT_NOK_WITH_MSG(copied->CopyWithoutTimeTravel({{Options::IGNORE_DELETE, "false"}}), + "ignore-delete"); + ASSERT_OK(table->CopyWithoutTimeTravel({{Options::IGNORE_DELETE, "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ScanContextBuilder().Finish()); ASSERT_NOK_WITH_MSG( TableScan::Create(load, std::move(context), {{Options::BUCKET_KEY, "b"}}), @@ -431,7 +436,7 @@ TEST(TableTest, TestScanWithMainBranchAlias) { ASSERT_OK_AND_ASSIGN(std::unique_ptr loaded_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, - table->Copy(dynamic_options)); + table->CopyWithoutTimeTravel(dynamic_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr loaded_scan, configured_table_1->NewScan(std::move(loaded_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr loaded_plan, loaded_scan->CreatePlan()); @@ -518,7 +523,7 @@ TEST(TableTest, TestGetTableDispatchesByTableType) { context_builder.EnablePreciseFileIndexEvaluate(true).Finish()); auto read = direct ? ([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - system_table->Copy(context_builder_options)); + system_table->CopyWithoutTimeTravel(context_builder_options)); return configured->NewRead(std::move(context)); })() : TableRead::Create(context_builder_load_context, std::move(context), @@ -582,7 +587,7 @@ TEST(TableTest, TestNewScanCreatesATableScan) { ScanContextBuilder ctx_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_3, - table->Copy(ctx_builder_options)); + table->CopyWithoutTimeTravel(ctx_builder_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, configured_table_3->NewScan(std::move(ctx))); } @@ -603,7 +608,7 @@ TEST(TableTest, TestNewReadCreatesATableRead) { ReadContextBuilder ctx_builder_18211; ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder_18211.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_4, - table->Copy(ctx_builder_18211_options)); + table->CopyWithoutTimeTravel(ctx_builder_18211_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, configured_table_4->NewRead(std::move(ctx))); } @@ -2406,7 +2411,7 @@ TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { WriteContextBuilder ctx_builder_18974("commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder_18974.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_5, - table->Copy(ctx_builder_18974_options)); + table->CopyWithoutTimeTravel(ctx_builder_18974_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, configured_table_5->NewWrite(std::move(ctx))); } @@ -2427,7 +2432,7 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { CommitContextBuilder ctx_builder_19886("commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ctx_builder_19886.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_6, - table->Copy(ctx_builder_19886_options)); + table->CopyWithoutTimeTravel(ctx_builder_19886_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, configured_table_6->NewCommit(std::move(ctx))); } @@ -2520,14 +2525,14 @@ TEST(TableTest, TestLoadedSchemaAndPathAreUsedWithoutReloading) { ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.SetReadFieldNames({"id"}).Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_7, - table->Copy(read_context_builder_options)); + table->CopyWithoutTimeTravel(read_context_builder_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, configured_table_7->NewRead(std::move(read_context))); std::map scan_context_builder_options; ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_8, - table->Copy(scan_context_builder_options)); + table->CopyWithoutTimeTravel(scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, configured_table_8->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); @@ -2548,9 +2553,9 @@ TEST(TableTest, TestContextOptionsCannotChangeLoadedTypeOrBranch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder_25016.Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr
configured, - table->Copy(scan_context_builder_25016_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->CopyWithoutTimeTravel( + scan_context_builder_25016_options)); return configured->NewScan(std::move(scan_context)); })(), "contradicts the loaded table"); @@ -2560,9 +2565,9 @@ TEST(TableTest, TestContextOptionsCannotChangeLoadedTypeOrBranch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder_25926.Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr
configured, - table->Copy(read_context_builder_25926_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, + table->CopyWithoutTimeTravel( + read_context_builder_25926_options)); return configured->NewRead(std::move(read_context)); })(), "contradicts the loaded table"); @@ -2572,8 +2577,9 @@ TEST(TableTest, TestContextOptionsCannotChangeLoadedTypeOrBranch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_context_builder.Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(write_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->CopyWithoutTimeTravel(write_context_builder_options)); return configured->NewWrite(std::move(write_context)); })(), "contradicts the loaded table"); @@ -2583,8 +2589,9 @@ TEST(TableTest, TestContextOptionsCannotChangeLoadedTypeOrBranch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_context_builder.Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(commit_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->CopyWithoutTimeTravel(commit_context_builder_options)); return configured->NewCommit(std::move(commit_context)); })(), "contradicts the loaded table"); @@ -2606,8 +2613,9 @@ TEST(TableTest, TestOperationOptionsDoNotMutateTheLoadedTable) { ASSERT_OK_AND_ASSIGN(std::unique_ptr invalid_context, invalid_context_builder.Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(invalid_context_builder_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->CopyWithoutTimeTravel(invalid_context_builder_options)); return configured->NewScan(std::move(invalid_context)); })(), "Cannot change bucket from or to -1 dynamically"); @@ -2616,7 +2624,7 @@ TEST(TableTest, TestOperationOptionsDoNotMutateTheLoadedTable) { ScanContextBuilder context_builder_27781; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_27781.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_14, - table->Copy(context_builder_27781_options)); + table->CopyWithoutTimeTravel(context_builder_27781_options)); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, configured_table_14->NewScan(std::move(context))); } @@ -2654,7 +2662,7 @@ TEST(TableTest, TestGlobalSystemTableRetainsCatalog) { ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder_28121.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_15, - table->Copy(scan_context_builder_28121_options)); + table->CopyWithoutTimeTravel(scan_context_builder_28121_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, configured_table_15->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); @@ -2663,7 +2671,7 @@ TEST(TableTest, TestGlobalSystemTableRetainsCatalog) { ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder_29694.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_16, - table->Copy(read_context_builder_29694_options)); + table->CopyWithoutTimeTravel(read_context_builder_29694_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, configured_table_16->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); @@ -2706,12 +2714,12 @@ TEST(TableTest, TestMetadataSystemTablesRejectConflictingBranches) { ScanContextBuilder().Finish()); auto read = ([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(read_context_options_82)); + table->CopyWithoutTimeTravel(read_context_options_82)); return configured->NewRead(std::move(read_context)); })(); auto scan = ([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(scan_context_options_83)); + table->CopyWithoutTimeTravel(scan_context_options_83)); return configured->NewScan(std::move(scan_context)); })(); if (requested == branch) { @@ -2740,7 +2748,7 @@ TEST(TableTest, TestOperatorFactoryPreservesLoadedSchemaCache) { ASSERT_OK(managed->GetSchemaManager()->ReadSchema(0)); ASSERT_OK(managed->GetSchemaManager()->ReadSchema(0)); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, - table->Copy({{Options::TARGET_FILE_ROW_NUM, "1"}})); + table->CopyWithoutTimeTravel({{Options::TARGET_FILE_ROW_NUM, "1"}})); auto copied = checked_pointer_cast(copy); ASSERT_EQ(managed->GetSchemaManager(), copied->GetSchemaManager()); ASSERT_OK(copied->GetSchemaManager()->ReadSchema(0)); @@ -2773,7 +2781,7 @@ TEST(TableTest, TestMainBranchAliasesUseMainSchemaForAllOperatorEntryPoints) { scan, TableScan::Create(builder_load, std::move(context), builder_options)); } else { ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_19, - table->Copy(builder_options)); + table->CopyWithoutTimeTravel(builder_options)); ASSERT_OK_AND_ASSIGN(scan, configured_table_19->NewScan(std::move(context))); } ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); @@ -2792,7 +2800,7 @@ TEST(TableTest, TestMainBranchAliasesUseMainSchemaForAllOperatorEntryPoints) { read_builder_options)); } else { ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_20, - table->Copy(read_builder_options)); + table->CopyWithoutTimeTravel(read_builder_options)); ASSERT_OK_AND_ASSIGN(read, configured_table_20->NewRead(std::move(read_context))); } @@ -2810,7 +2818,7 @@ TEST(TableTest, TestMainBranchAliasesUseMainSchemaForAllOperatorEntryPoints) { write_builder_options)); } else { ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_21, - table->Copy(write_builder_options)); + table->CopyWithoutTimeTravel(write_builder_options)); ASSERT_OK_AND_ASSIGN(write, configured_table_21->NewWrite(std::move(write_context))); } @@ -2830,7 +2838,7 @@ TEST(TableTest, TestMainBranchAliasesUseMainSchemaForAllOperatorEntryPoints) { commit_builder_options)); } else { ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_22, - table->Copy(commit_builder_options)); + table->CopyWithoutTimeTravel(commit_builder_options)); ASSERT_OK_AND_ASSIGN(commit, configured_table_22->NewCommit(std::move(commit_context))); } @@ -2864,12 +2872,12 @@ TEST(TableTest, TestExplicitContextBranchMustMatchLoadedTable) { ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); auto write = ([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(write_builder_options)); + table->CopyWithoutTimeTravel(write_builder_options)); return configured->NewWrite(std::move(write_context)); })(); auto read = ([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(read_builder_options)); + table->CopyWithoutTimeTravel(read_builder_options)); return configured->NewRead(std::move(read_context)); })(); if (requested && BranchManager::NormalizeBranch(*requested) != table_branch) { @@ -2947,7 +2955,7 @@ TEST(TableTest, TestPreloadedSchemaBranchMatchesScanAndRead) { ASSERT_EQ(expected_options, std::dynamic_pointer_cast(loaded->GetSchema())->Options()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_25, - loaded->Copy(read_context_builder_34363_options)); + loaded->CopyWithoutTimeTravel(read_context_builder_34363_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, configured_table_25->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, @@ -3010,7 +3018,7 @@ TEST(TableTest, TestLoadedTableReadRetainsLoadSchemeMap) { std::unique_ptr read, direct ? ([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(context_builder_36442_options)); + table->CopyWithoutTimeTravel(context_builder_36442_options)); return configured->NewRead(std::move(context)); })() : TableRead::Create(context_builder_36442_load_context, std::move(context), @@ -3052,7 +3060,7 @@ TEST(TableTest, TestReadRetainsLoadSchemeMapForMainAndFallback) { } ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_27, - table->Copy(builder_options)); + table->CopyWithoutTimeTravel(builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, configured_table_27->NewRead(std::move(context))); std::vector reads; @@ -3077,7 +3085,7 @@ TEST(TableTest, TestReadRetainsLoadSchemeMapForMainAndFallback) { ReadContextBuilder context_builder_38424; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_38424.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_28, - table->Copy(context_builder_38424_options)); + table->CopyWithoutTimeTravel(context_builder_38424_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, configured_table_28->NewRead(std::move(context))); auto* append_read = dynamic_cast(read.get()); @@ -3126,7 +3134,7 @@ TEST(TableTest, TestSystemTablePathRetainsSchemeMapForDependencyReads) { context_builder_41670_options = context_builder_42682_options; ASSERT_OK_AND_ASSIGN(context, context_builder_42682.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_29, - table->Copy(context_builder_41670_options)); + table->CopyWithoutTimeTravel(context_builder_41670_options)); ASSERT_OK_AND_ASSIGN(read, configured_table_29->NewRead(std::move(context))); } else { ASSERT_OK_AND_ASSIGN( @@ -3158,7 +3166,7 @@ TEST(TableTest, TestSystemTableReadRetainsLoadSchemeMapWithoutReplacingFileSyste ReadContextBuilder context_builder_43111; ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder_43111.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_30, - table->Copy(context_builder_43111_options)); + table->CopyWithoutTimeTravel(context_builder_43111_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, configured_table_30->NewRead(std::move(context))); auto* append_read = dynamic_cast(read.get()); @@ -3201,7 +3209,7 @@ TEST(TableTest, TestRealtimeFallbackOnlyAllowsMainBranch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); auto read = ([&]() -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(builder_options)); + table->CopyWithoutTimeTravel(builder_options)); return configured->NewRead(std::move(context)); })(); if (realtime_enabled && fallback != "main") { @@ -3348,7 +3356,7 @@ TEST(TableTest, LoadedFallbackReadUsesBoundStorage) { CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl", "dev"), "dev")); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_32, - dev->Copy({{Options::SCAN_FALLBACK_BRANCH, "main"}})); + dev->CopyWithoutTimeTravel({{Options::SCAN_FALLBACK_BRANCH, "main"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, configured_table_32->NewRead(std::move(context))); auto* fallback = dynamic_cast(read.get()); @@ -3443,8 +3451,9 @@ TEST(TableTest, LoadedHandlesCheckNewAuthorizationSettings) { ASSERT_OK_AND_ASSIGN(std::unique_ptr enabled, ReadContextBuilder().Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - view->Copy(enabled_options_100)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + view->CopyWithoutTimeTravel(enabled_options_100)); return configured->NewRead(std::move(enabled)); })(), "query-auth.enabled"); @@ -3463,8 +3472,9 @@ TEST(TableTest, LoadedHandlesCheckNewAuthorizationSettings) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, ReadContextBuilder().Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - view->Copy(context_options_101)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + view->CopyWithoutTimeTravel(context_options_101)); return configured->NewRead(std::move(context)); })(), "query-auth.enabled"); @@ -3474,7 +3484,7 @@ TEST(TableTest, LoadedHandlesCheckNewAuthorizationSettings) { read_options_102[Options::BRANCH] = branch; ASSERT_OK_AND_ASSIGN(std::unique_ptr read, ReadContextBuilder().Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_35, - table->Copy(read_options_102)); + table->CopyWithoutTimeTravel(read_options_102)); ASSERT_OK_AND_ASSIGN(std::unique_ptr guarded, configured_table_35->NewRead(std::move(read))); ASSERT_NE(nullptr, dynamic_cast(guarded.get())); @@ -3484,8 +3494,9 @@ TEST(TableTest, LoadedHandlesCheckNewAuthorizationSettings) { scan_options_103[Options::BRANCH] = branch; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, ScanContextBuilder().Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy(scan_options_103)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->CopyWithoutTimeTravel(scan_options_103)); return configured->NewScan(std::move(scan)); })(), "without the catalog"); @@ -3697,16 +3708,16 @@ TEST(TableTest, TestCatalogOnlyLoadsRetainTheResolvedBranchSchema) { } ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().SetReadFieldNames({"remote"}).Finish()); - ASSERT_OK(TableRead::Create(load, std::move(read_context))); + ASSERT_OK(TableRead::Create(load, std::move(read_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder().Finish()); - ASSERT_OK(TableScan::Create(load, std::move(scan_context))); + ASSERT_OK(TableScan::Create(load, std::move(scan_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, WriteContextBuilder("writer").Finish()); - ASSERT_OK(FileStoreWrite::Create(load, std::move(write_context))); + ASSERT_OK(FileStoreWrite::Create(load, std::move(write_context), {})); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, CommitContextBuilder("writer").Finish()); - ASSERT_OK(FileStoreCommit::Create(load, std::move(commit_context))); + ASSERT_OK(FileStoreCommit::Create(load, std::move(commit_context), {})); } } } @@ -3731,8 +3742,9 @@ TEST(TableTest, TestLoadContextValidateExpectedCatalogUuid) { ASSERT_OK_AND_ASSIGN(std::unique_ptr context, CommitContextBuilder("writer").Finish()); ASSERT_NOK_WITH_MSG(([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured, - table->Copy({{Options::BRANCH, "dev"}})); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->CopyWithoutTimeTravel({{Options::BRANCH, "dev"}})); return configured->NewCommit(std::move(context)); })(), "branch contradicts"); diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index a8e5f6739..89da8d99b 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -895,7 +895,7 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsFactoryCatalog) { ScanContextBuilder scan_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_1, - table->Copy(scan_context_builder_options)); + table->CopyWithoutTimeTravel(scan_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, configured_table_1->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); @@ -903,7 +903,7 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsFactoryCatalog) { ReadContextBuilder read_context_builder; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured_table_2, - table->Copy(read_context_builder_options)); + table->CopyWithoutTimeTravel(read_context_builder_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, configured_table_2->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); @@ -1246,7 +1246,7 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { load.identifier.emplace(branch_identifier); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, ReadContextBuilder().SetReadFieldNames({"b0"}).Finish()); - ASSERT_OK(TableRead::Create(load, std::move(read_context))); + ASSERT_OK(TableRead::Create(load, std::move(read_context), {})); // the default branch is addressed as the bare table: "t1$branch_main" resolves to // "t1" and carries no branch option diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 53da9fd35..757f72c25 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -996,7 +996,7 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTablesAtPinnedSnapshot) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder_0.Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_1, - table->Copy(scan_context_builder_0_options)); + table->CopyWithoutTimeTravel(scan_context_builder_0_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, configured_table_1->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); @@ -1005,8 +1005,9 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTablesAtPinnedSnapshot) { read_context_builder_48503_options = read_options; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder_48503.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_2, - table->Copy(read_context_builder_48503_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured_table_2, + table->CopyWithoutTimeTravel(read_context_builder_48503_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, configured_table_2->NewRead(std::move(read_context))); return read->CreateReader(plan->Splits()); @@ -1893,16 +1894,17 @@ TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadO scan_context_builder_48980_options = read_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder_48980.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - direct ? ([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr
configured, - table->Copy(scan_context_builder_48980_options)); - return configured->NewScan(std::move(scan_context)); - })() - : TableScan::Create(scan_context_builder_48980_load_context, - std::move(scan_context), - scan_context_builder_48980_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + direct ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->CopyWithoutTimeTravel(scan_context_builder_48980_options)); + return configured->NewScan(std::move(scan_context)); + })() + : TableScan::Create(scan_context_builder_48980_load_context, + std::move(scan_context), + scan_context_builder_48980_options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); TableLoadContext read_context_builder_93839_load_context{path}; std::map read_context_builder_93839_options; @@ -1910,16 +1912,17 @@ TEST(SystemTableReadInteTest, TestChangelogSequenceNumberCannotBeDisabledByReadO read_context_builder_93839_options = read_options; ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder_93839.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - direct ? ([&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr
configured, - table->Copy(read_context_builder_93839_options)); - return configured->NewRead(std::move(read_context)); - })() - : TableRead::Create(read_context_builder_93839_load_context, - std::move(read_context), - read_context_builder_93839_options)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + direct ? ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr
configured, + table->CopyWithoutTimeTravel(read_context_builder_93839_options)); + return configured->NewRead(std::move(read_context)); + })() + : TableRead::Create(read_context_builder_93839_load_context, + std::move(read_context), + read_context_builder_93839_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1958,7 +1961,8 @@ TEST(SystemTableReadInteTest, TestCopiedChangelogHonorsDynamicSequenceNumber) { SCOPED_TRACE(kind); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("foo", "bar$" + kind))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured, table->Copy(read_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
configured, + table->CopyWithoutTimeTravel(read_options)); ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_schema, configured->GetSchema()->GetArrowSchema()); auto copied = arrow::ImportSchema(copied_schema.get()); @@ -4716,7 +4720,7 @@ Result> ReadGlobalSystemTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_7, - system_table->Copy(scan_context_builder_options)); + system_table->CopyWithoutTimeTravel(scan_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, configured_table_7->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); @@ -4728,7 +4732,7 @@ Result> ReadGlobalSystemTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
configured_table_8, - system_table->Copy(read_context_builder_options)); + system_table->CopyWithoutTimeTravel(read_context_builder_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, configured_table_8->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, From 2a1546d9b07272320d0af5dca60f50783c9bc7d5 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Sat, 14 Nov 2026 08:27:24 +0800 Subject: [PATCH 6/6] fix(table): retain inferred scan modes across copies --- docs/source/user_guide/catalog.rst | 4 +- src/paimon/core/schema/schema_validation.cpp | 15 ++++++ .../core/schema/schema_validation_test.cpp | 41 ++++++++++++++ src/paimon/core/table/table_test.cpp | 53 +++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index c6ddba5d8..3d74ba422 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -163,7 +163,9 @@ storage binding and supplied metadata. Operation Contexts hold execution setting finishing an operation Context performs no table metadata I/O. ``CopyWithoutTimeTravel`` preserves the bound table schema, matching Java's ``copyWithoutTimeTravel``. It does not resolve a historical schema from snapshot -or timestamp options: +or timestamp options. When ``scan.mode`` is absent, the copy stores Java-compatible +scan-mode defaults inferred from the options. Later copies retain that mode unless +it is explicitly overridden: .. code-block:: cpp diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 8312ed76c..35e4921f6 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -326,6 +326,21 @@ Result> SchemaValidation::MergeDynamicOptions } options[key] = value; } + if (!options.count(Options::SCAN_MODE)) { + // Java CoreOptions.setDefaultValues: the first matching rule wins. + if (options.count(Options::SCAN_TIMESTAMP_MILLIS) || + options.count(Options::SCAN_TIMESTAMP)) { + options[Options::SCAN_MODE] = "from-timestamp"; + } else if (options.count("scan.file-creation-time-millis")) { + options[Options::SCAN_MODE] = "from-file-creation-time"; + } else if (options.count(Options::SCAN_SNAPSHOT_ID)) { + options[Options::SCAN_MODE] = "from-snapshot"; + } else if (options.count("incremental-between-timestamp") || + options.count("incremental-between") || + options.count("incremental-to-auto-tag")) { + options[Options::SCAN_MODE] = "incremental"; + } + } return options; } diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 3f7bb3b5f..0c9b14044 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -2158,6 +2158,47 @@ TEST(SchemaValidationTest, TestMapSharedShreddingRejectsPostponeBucketMode) { "MAP shared-shredding currently does not support postpone bucket mode."); } +TEST(SchemaValidationTest, TestMaterializeScanModeDefaults) { + using OptionsMap = std::map; + const std::vector>> cases = { + {{}, std::nullopt}, + {{{Options::SCAN_TIMESTAMP_MILLIS, "1000"}}, "from-timestamp"}, + {{{Options::SCAN_TIMESTAMP, "2023-06-01"}}, "from-timestamp"}, + {{{"scan.file-creation-time-millis", "1000"}}, "from-file-creation-time"}, + {{{Options::SCAN_SNAPSHOT_ID, "1"}}, "from-snapshot"}, + {{{"incremental-between-timestamp", "1000,2000"}}, "incremental"}, + {{{"incremental-between", "1,2"}}, "incremental"}, + {{{"incremental-to-auto-tag", "daily"}}, "incremental"}, + {{{Options::SCAN_TAG_NAME, "tag1"}}, std::nullopt}, + {{{Options::SCAN_TIMESTAMP_MILLIS, "1000"}, {Options::SCAN_SNAPSHOT_ID, "1"}}, + "from-timestamp"}, + {{{"scan.file-creation-time-millis", "1000"}, {Options::SCAN_SNAPSHOT_ID, "1"}}, + "from-file-creation-time"}, + {{{Options::SCAN_SNAPSHOT_ID, "1"}, {"incremental-between", "1,2"}}, "from-snapshot"}, + {{{Options::SCAN_TIMESTAMP_MILLIS, "1000"}, {Options::SCAN_MODE, "latest-full"}}, + "latest-full"}, + {{{Options::SCAN_TIMESTAMP_MILLIS, "1000"}, {Options::SCAN_MODE, "default"}}, "default"}, + {{{Options::SCAN_TIMESTAMP_MILLIS, "1000"}, {Options::SCAN_MODE, ""}}, ""}, + }; + auto arrow_schema = arrow::schema({arrow::field("a", arrow::int32())}); + for (size_t i = 0; i < cases.size(); ++i) { + SCOPED_TRACE(i); + auto options = cases[i].first; + auto expected = options; + if (cases[i].second) { + expected[Options::SCAN_MODE] = *cases[i].second; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + TableSchema::Create(0, arrow_schema, {}, {}, options)); + ASSERT_OK_AND_ASSIGN(OptionsMap merged, SchemaValidation::MergeDynamicOptions(*schema, {})); + ASSERT_EQ(merged, expected); + ASSERT_OK_AND_ASSIGN(schema, schema->Copy(merged)); + ASSERT_OK_AND_ASSIGN(OptionsMap repeated, + SchemaValidation::MergeDynamicOptions(*schema, {})); + ASSERT_EQ(repeated, expected); + } +} + TEST(SchemaValidationTest, TestCopyImmutableOptions) { const std::map> changes = { {"bucket-key", {"a", "b"}}, diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 957a4fbc3..5655caa19 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -178,6 +178,59 @@ TEST(TableTest, TestCopyRetainsBindingsAndMergesOptionsIndependently) { ASSERT_TRUE(weak_catalog.expired()); } +TEST(TableTest, TestChainedCopiesRetainMaterializedScanMode) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), Identifier("db", "tbl"))); + const auto original_options = checked_pointer_cast(table->GetSchema())->Options(); + using OptionsMap = std::map; + const std::vector> cases = { + {{{Options::SCAN_TIMESTAMP_MILLIS, "1000"}}, "from-timestamp"}, + {{{Options::SCAN_TIMESTAMP, "2023-06-01"}}, "from-timestamp"}, + {{{Options::SCAN_SNAPSHOT_ID, "1"}}, "from-snapshot"}, + {{{Options::SCAN_TIMESTAMP_MILLIS, "1000"}, {Options::SCAN_MODE, "latest-full"}}, + "latest-full"}, + }; + for (const auto& [first_options, expected_mode] : cases) { + SCOPED_TRACE(expected_mode); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
first, + table->CopyWithoutTimeTravel(first_options)); + const auto& first_schema_options = + checked_pointer_cast(first->GetSchema())->Options(); + ASSERT_EQ(first_schema_options.at(Options::SCAN_MODE), expected_mode); + // A later snapshot option must not change a mode inferred by an earlier copy. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
second, + first->CopyWithoutTimeTravel({{Options::SCAN_SNAPSHOT_ID, "2"}})); + const auto& second_options = + checked_pointer_cast(second->GetSchema())->Options(); + ASSERT_EQ(second_options.at(Options::SCAN_MODE), expected_mode); + ASSERT_EQ(second_options.at(Options::SCAN_SNAPSHOT_ID), "2"); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(second_options, fs)); + ASSERT_EQ(core_options.GetStartupMode().ToString(), expected_mode); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
third, + second->CopyWithoutTimeTravel({{Options::SCAN_MODE, "latest"}})); + ASSERT_EQ( + checked_pointer_cast(third->GetSchema())->Options().at(Options::SCAN_MODE), + "latest"); + ASSERT_EQ(first_schema_options.at(Options::SCAN_MODE), expected_mode); + ASSERT_EQ(checked_pointer_cast(table->GetSchema())->Options(), + original_options); + } + // A supplied schema already containing a selector receives the same defaults on an empty copy. + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + checked_pointer_cast(table->GetSchema()) + ->MergeOptions({{Options::SCAN_TIMESTAMP_MILLIS, "1000"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr loaded, + FileStoreTable::Create(fs, dir->Str(), schema, Identifier("db", "tbl"), "", "main")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, loaded->CopyWithoutTimeTravel({})); + ASSERT_EQ( + checked_pointer_cast(copied->GetSchema())->Options().at(Options::SCAN_MODE), + "from-timestamp"); +} + TEST(TableTest, TestCopyRejectsBucketKeyChangesWithoutLosingRows) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir);