diff --git a/docs/docs/pypaimon/branches.md b/docs/docs/pypaimon/branches.md index 59173fa61d80..803ff1764f0e 100644 --- a/docs/docs/pypaimon/branches.md +++ b/docs/docs/pypaimon/branches.md @@ -32,6 +32,10 @@ A branch maintains a separate table history. Creating a branch without a tag copies the current schema into an empty branch. To include existing data, create the branch from a retained tag. +Use `table.copy({'branch': 'dev'})` to read the latest state of `dev` with that +branch's current schema. When you also select a snapshot or tag, the read uses +the schema recorded for that snapshot or tag. + :::info PyPaimon provides two implementations of `BranchManager`: diff --git a/paimon-python/pypaimon/table/file_store_table.py b/paimon-python/pypaimon/table/file_store_table.py index d0c3ff65c9de..860307454f34 100644 --- a/paimon-python/pypaimon/table/file_store_table.py +++ b/paimon-python/pypaimon/table/file_store_table.py @@ -452,7 +452,46 @@ def drop_global_index(self, index_column, index_type: str = "btree", ) def copy(self, options: dict) -> 'FileStoreTable': - return self._copy(options, resolve_time_travel=True) + table = self._copy(options, resolve_time_travel=True) + if table.identifier == self.identifier: + return table + + from pypaimon.snapshot.time_travel_util import SCAN_KEYS + if any(key in table.table_schema.options for key in SCAN_KEYS): + return table + + env = table.catalog_environment + if env.rest_table_response is not None: + from pypaimon.api.api_response import ErrorResponse + from pypaimon.catalog.catalog_exception import TableNotExistException + + loader = env.catalog_loader + if loader is None: + from pypaimon.catalog.rest.rest_catalog_loader import RESTCatalogLoader + loader = RESTCatalogLoader(env.catalog_context()) + try: + schema = loader.load().load_table_metadata(table.identifier).schema + except TableNotExistException as exc: + if getattr(exc.__cause__, 'resource_type', None) != ErrorResponse.RESOURCE_TYPE_BRANCH: + raise + # The copy can carry a branch identifier before that branch exists. + return table + location = schema.options.get(CoreOptions.PATH.key()) + if location != table.table_path: + from pypaimon.common.file_io import _is_local_path + from pypaimon.filesystem.local_file_io import LocalFileIO + + if not (location and _is_local_path(location) and _is_local_path(table.table_path) + and LocalFileIO._to_file(location) == LocalFileIO._to_file(table.table_path)): + raise ValueError("REST resolved a different table location. Reopen the table.") + else: + schema = table.schema_manager.latest() + if schema is None: + return table + + # Refresh the schema while retaining the copy's FileIO and catalog binding. + target = FileStoreTable(table.file_io, table.identifier, table.table_path, schema, env) + return target._copy(table._applied_dynamic_options, resolve_time_travel=True) def copy_without_time_travel(self, options: dict) -> 'FileStoreTable': """Copy this table while preserving its already resolved schema.""" diff --git a/paimon-python/pypaimon/tests/multimodal_columns_test.py b/paimon-python/pypaimon/tests/multimodal_columns_test.py index 928fd2ff51a0..4ebfd071c8d9 100644 --- a/paimon-python/pypaimon/tests/multimodal_columns_test.py +++ b/paimon-python/pypaimon/tests/multimodal_columns_test.py @@ -991,41 +991,48 @@ def test_update_columns_by_row_id_checks_current_branch_schema_and_keeps_dynamic @pytest.mark.parametrize("method", ["batch", "predicate"]) -def test_row_id_updates_reject_branch_copy_with_same_schema_id_and_different_field_ids(frames, method): +def test_row_id_updates_reject_recreated_branch_with_same_schema_id_and_different_field_ids(frames, method): from pypaimon.schema.data_types import AtomicType from pypaimon.schema.schema_change import Move, SchemaChange connection = pmm.MultimodalConnection(frames.catalog) frames.catalog.create_tag(frames.identifier, "base", _snapshot(frames)) frames.catalog.create_branch(frames.identifier, "dev", tag_name="base") - frames.catalog.alter_table(frames.identifier, [SchemaChange.set_option("read.batch-size", "3")]) identifier = frames.identifier + "$branch_dev" + frames.catalog.alter_table(identifier, [SchemaChange.set_option("read.batch-size", "3")]) + stale = connection.get_table(identifier) + + frames.catalog.drop_branch(frames.identifier, "dev") + frames.catalog.create_branch(frames.identifier, "dev", tag_name="base") frames.catalog.alter_table(identifier, [ SchemaChange.drop_column("label"), SchemaChange.add_column("label", AtomicType("STRING"), move=Move.before("label", "image"))]) - copied = connection.get_table(frames.identifier).copy({"branch": "dev"}) current = connection.get_table(identifier) - assert copied.raw_table.table_schema.id == current.raw_table.table_schema.id - assert copied.raw_table.field_names == current.raw_table.field_names - assert [field.type for field in copied.raw_table.fields] == [field.type for field in current.raw_table.fields] - assert copied.raw_table.field_dict["label"].id != current.raw_table.field_dict["label"].id + assert stale.raw_table.table_schema.id == current.raw_table.table_schema.id + assert stale.raw_table.field_names == current.raw_table.field_names + assert [field.type for field in stale.raw_table.fields] == [field.type for field in current.raw_table.fields] + assert stale.raw_table.field_dict["label"].id != current.raw_table.field_dict["label"].id snapshot, files = _snapshot(current), set(_files(current)) with patch("pypaimon.table.file_store_table.FileStoreTable.new_batch_write_builder", side_effect=AssertionError("must not stage files")): with pytest.raises(ValueError, match="schema has changed.*Reopen"): if method == "batch": - copied.update_columns_by_row_id(pa.table({"_ROW_ID": [0], "label": ["new"]}), columns=["label"]) + stale.update_columns_by_row_id(pa.table({"_ROW_ID": [0], "label": ["new"]}), columns=["label"]) else: - copied.update("_ROW_ID = 0", {"label": "new"}) + stale.update("_ROW_ID = 0", {"label": "new"}) assert _snapshot(current) == snapshot assert set(_files(current)) == files assert current.scan().select(["label"]).to_list() == [{"label": None}] * 5 + + copied = connection.get_table(frames.identifier).copy({"branch": "dev"}) + assert copied.raw_table.fields == current.raw_table.fields if method == "batch": - current.update_columns_by_row_id(pa.table({"_ROW_ID": [0], "label": ["new"]}), columns=["label"]) + copied.update_columns_by_row_id(pa.table({"_ROW_ID": [0], "label": ["new"]}), columns=["label"]) else: - current.update("_ROW_ID = 0", {"label": "new"}) + copied.update("_ROW_ID = 0", {"label": "new"}) assert _rows(connection.get_table(identifier), ["id", "label"]) == [ {"id": i, "label": "new" if i == 0 else None} for i in range(5)] + assert connection.get_table(frames.identifier).scan().select(["label"]).to_list() == [{"label": "old"}] * 5 def test_business_predicate_with_row_id_string_literal_does_not_check_schema(frames): diff --git a/paimon-python/pypaimon/tests/table/time_travel_schema_evolution_test.py b/paimon-python/pypaimon/tests/table/time_travel_schema_evolution_test.py index 0db572e0cb3a..5e4c69f33453 100644 --- a/paimon-python/pypaimon/tests/table/time_travel_schema_evolution_test.py +++ b/paimon-python/pypaimon/tests/table/time_travel_schema_evolution_test.py @@ -162,6 +162,93 @@ def test_branch_time_travel_back_to_main(self, branch): {'id': 2, 'payload': 'main', 'source': 'web'}, ] + def test_branch_copy_reads_latest_schema(self): + self._create_branched_table([ + DataField(0, 'id', AtomicType('INT')), + DataField(1, 'count', AtomicType('INT')), + ], {'id': 1, 'count': 1}) + self.catalog.alter_table('test.t', [ + SchemaChange.add_column('source', AtomicType('STRING'))]) + self.catalog.alter_table('test.t$branch_dev', [ + SchemaChange.update_column_type('count', AtomicType('BIGINT'))]) + main = self.catalog.get_table('test.t') + dev = self.catalog.get_table('test.t$branch_dev') + assert main.table_schema.id == dev.table_schema.id + self._write(dev, {'id': 2, 'count': 3000000000}) + + switched = main.copy({'branch': 'dev'}) + assert sorted(self._read(switched), key=lambda row: row['id']) == [ + {'id': 1, 'count': 1}, {'id': 2, 'count': 3000000000}] + assert switched.copy({'branch': None}).fields == main.fields + + self.catalog.alter_table('test.t$branch_dev', [ + SchemaChange.add_column('quality', AtomicType('STRING'))]) + latest = main.copy({'branch': 'dev'}) + assert latest.field_names == ['id', 'count', 'quality'] + assert self._read(latest.copy({'scan.tag-name': 'base'})) == [{'id': 1, 'count': 1}] + row = {'id': 3, 'count': 4000000000, 'quality': 'verified'} + self._write(latest, row) + assert row in self._read(self.catalog.get_table('test.t$branch_dev')) + assert self._read(main) == [{'id': 1, 'count': 1, 'source': None}] + + def test_branch_copy_applies_dynamic_options_to_branch_defaults(self): + main = self._create_branched_table([DataField(0, 'id', AtomicType('INT'))], {'id': 1}) + self.catalog.alter_table('test.t$branch_dev', [ + SchemaChange.set_option('file.compression', 'zstd'), + SchemaChange.set_option('read.batch-size', '17'), + SchemaChange.set_option('read.parallelism', '3')]) + + configured = main.copy({'read.batch-size': '7', 'read.parallelism': None}) + switched = configured.copy({'branch': 'dev'}) + assert switched.table_schema.options['file.compression'] == 'zstd' + assert switched.table_schema.options['read.batch-size'] == '7' + assert 'read.parallelism' not in switched.table_schema.options + assert main.table_schema.options['file.compression'] == 'none' + + @pytest.mark.parametrize('external', [False, True]) + def test_branch_copy_loads_rest_branch_metadata(self, native_rest_catalog, external): + from pypaimon.common.file_io import FileIO + from pypaimon.schema.schema_manager import SchemaManager + + catalog = native_rest_catalog + name = 'default.branch_copy' + fields = [DataField(0, 'id', AtomicType('INT'))] + options = {} + if external: + path = (self.root / 'warehouse' / 'default' / 'branch_copy').as_uri() + SchemaManager(FileIO.get(path), path).create_table(Schema(fields=fields)) + options['path'] = path + catalog.create_table(name, Schema(fields=fields, options=options), False) + main = catalog.get_table(name) + assert main.copy({'branch': 'dev'}).current_branch() == 'dev' + catalog.create_branch(name, 'dev') + catalog.alter_table(name + '$branch_dev', [ + SchemaChange.add_column('quality', AtomicType('STRING'))]) + + direct = catalog.get_table(name + '$branch_dev') + switched = main.copy({'branch': 'dev'}) + assert switched.fields == direct.fields + assert switched.table_schema.options['path'] == direct.table_schema.options['path'] + assert switched.file_io is main.file_io + assert switched.catalog_environment.identifier == direct.identifier + assert switched.catalog_environment.uuid is None + assert switched.catalog_environment.rest_table_response == main.catalog_environment.rest_table_response + + def test_branch_copy_rejects_relocated_rest_table(self, native_rest_catalog): + catalog = native_rest_catalog + name = 'default.relocated' + fields = [DataField(0, 'id', AtomicType('INT'))] + catalog.create_table(name, Schema(fields=fields), False) + catalog.create_branch(name, 'dev') + dev = catalog.get_table(name + '$branch_dev') + + catalog.drop_table(name) + location = (self.root / 'replacement').as_uri() + catalog.create_table(name, Schema(fields=fields, options={'path': location}), False) + assert catalog.get_table(name).table_path == location + with pytest.raises(ValueError, match='different table location'): + dev.copy({'branch': 'main'}) + def test_commit_retry_refreshes_snapshot_schema(self, monkeypatch): original = self._create([DataField(0, 'id', AtomicType('INT'))]) self._write(original, {'id': 1})