Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/docs/pypaimon/branches.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ A branch maintains a separate table history. Creating a branch without a tag
copies the current schema into an empty branch. To include existing data, create
the branch from a retained tag.

Use `table.copy({'branch': 'dev'})` to read the latest state of `dev` with that
branch's current schema. When you also select a snapshot or tag, the read uses
the schema recorded for that snapshot or tag.

:::info

PyPaimon provides two implementations of `BranchManager`:
Expand Down
41 changes: 40 additions & 1 deletion paimon-python/pypaimon/table/file_store_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -452,7 +452,46 @@ def drop_global_index(self, index_column, index_type: str = "btree",
)

def copy(self, options: dict) -> 'FileStoreTable':
return self._copy(options, resolve_time_travel=True)
table = self._copy(options, resolve_time_travel=True)
if table.identifier == self.identifier:
return table

from pypaimon.snapshot.time_travel_util import SCAN_KEYS
if any(key in table.table_schema.options for key in SCAN_KEYS):
return table

env = table.catalog_environment
if env.rest_table_response is not None:
from pypaimon.api.api_response import ErrorResponse
from pypaimon.catalog.catalog_exception import TableNotExistException

loader = env.catalog_loader
if loader is None:
from pypaimon.catalog.rest.rest_catalog_loader import RESTCatalogLoader
loader = RESTCatalogLoader(env.catalog_context())
try:
schema = loader.load().load_table_metadata(table.identifier).schema
except TableNotExistException as exc:
if getattr(exc.__cause__, 'resource_type', None) != ErrorResponse.RESOURCE_TYPE_BRANCH:
raise
# The copy can carry a branch identifier before that branch exists.
return table
location = schema.options.get(CoreOptions.PATH.key())
if location != table.table_path:
from pypaimon.common.file_io import _is_local_path
from pypaimon.filesystem.local_file_io import LocalFileIO

if not (location and _is_local_path(location) and _is_local_path(table.table_path)
and LocalFileIO._to_file(location) == LocalFileIO._to_file(table.table_path)):
raise ValueError("REST resolved a different table location. Reopen the table.")
else:
schema = table.schema_manager.latest()
if schema is None:
return table

# Refresh the schema while retaining the copy's FileIO and catalog binding.
target = FileStoreTable(table.file_io, table.identifier, table.table_path, schema, env)
return target._copy(table._applied_dynamic_options, resolve_time_travel=True)

def copy_without_time_travel(self, options: dict) -> 'FileStoreTable':
"""Copy this table while preserving its already resolved schema."""
Expand Down
29 changes: 18 additions & 11 deletions paimon-python/pypaimon/tests/multimodal_columns_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -991,41 +991,48 @@ def test_update_columns_by_row_id_checks_current_branch_schema_and_keeps_dynamic


@pytest.mark.parametrize("method", ["batch", "predicate"])
def test_row_id_updates_reject_branch_copy_with_same_schema_id_and_different_field_ids(frames, method):
def test_row_id_updates_reject_recreated_branch_with_same_schema_id_and_different_field_ids(frames, method):
from pypaimon.schema.data_types import AtomicType
from pypaimon.schema.schema_change import Move, SchemaChange

connection = pmm.MultimodalConnection(frames.catalog)
frames.catalog.create_tag(frames.identifier, "base", _snapshot(frames))
frames.catalog.create_branch(frames.identifier, "dev", tag_name="base")
frames.catalog.alter_table(frames.identifier, [SchemaChange.set_option("read.batch-size", "3")])
identifier = frames.identifier + "$branch_dev"
frames.catalog.alter_table(identifier, [SchemaChange.set_option("read.batch-size", "3")])
stale = connection.get_table(identifier)

frames.catalog.drop_branch(frames.identifier, "dev")
frames.catalog.create_branch(frames.identifier, "dev", tag_name="base")
frames.catalog.alter_table(identifier, [
SchemaChange.drop_column("label"),
SchemaChange.add_column("label", AtomicType("STRING"), move=Move.before("label", "image"))])
copied = connection.get_table(frames.identifier).copy({"branch": "dev"})
current = connection.get_table(identifier)
assert copied.raw_table.table_schema.id == current.raw_table.table_schema.id
assert copied.raw_table.field_names == current.raw_table.field_names
assert [field.type for field in copied.raw_table.fields] == [field.type for field in current.raw_table.fields]
assert copied.raw_table.field_dict["label"].id != current.raw_table.field_dict["label"].id
assert stale.raw_table.table_schema.id == current.raw_table.table_schema.id
assert stale.raw_table.field_names == current.raw_table.field_names
assert [field.type for field in stale.raw_table.fields] == [field.type for field in current.raw_table.fields]
assert stale.raw_table.field_dict["label"].id != current.raw_table.field_dict["label"].id
snapshot, files = _snapshot(current), set(_files(current))
with patch("pypaimon.table.file_store_table.FileStoreTable.new_batch_write_builder",
side_effect=AssertionError("must not stage files")):
with pytest.raises(ValueError, match="schema has changed.*Reopen"):
if method == "batch":
copied.update_columns_by_row_id(pa.table({"_ROW_ID": [0], "label": ["new"]}), columns=["label"])
stale.update_columns_by_row_id(pa.table({"_ROW_ID": [0], "label": ["new"]}), columns=["label"])
else:
copied.update("_ROW_ID = 0", {"label": "new"})
stale.update("_ROW_ID = 0", {"label": "new"})
assert _snapshot(current) == snapshot
assert set(_files(current)) == files
assert current.scan().select(["label"]).to_list() == [{"label": None}] * 5

copied = connection.get_table(frames.identifier).copy({"branch": "dev"})
assert copied.raw_table.fields == current.raw_table.fields
if method == "batch":
current.update_columns_by_row_id(pa.table({"_ROW_ID": [0], "label": ["new"]}), columns=["label"])
copied.update_columns_by_row_id(pa.table({"_ROW_ID": [0], "label": ["new"]}), columns=["label"])
else:
current.update("_ROW_ID = 0", {"label": "new"})
copied.update("_ROW_ID = 0", {"label": "new"})
assert _rows(connection.get_table(identifier), ["id", "label"]) == [
{"id": i, "label": "new" if i == 0 else None} for i in range(5)]
assert connection.get_table(frames.identifier).scan().select(["label"]).to_list() == [{"label": "old"}] * 5


def test_business_predicate_with_row_id_string_literal_does_not_check_schema(frames):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,93 @@ def test_branch_time_travel_back_to_main(self, branch):
{'id': 2, 'payload': 'main', 'source': 'web'},
]

def test_branch_copy_reads_latest_schema(self):
self._create_branched_table([
DataField(0, 'id', AtomicType('INT')),
DataField(1, 'count', AtomicType('INT')),
], {'id': 1, 'count': 1})
self.catalog.alter_table('test.t', [
SchemaChange.add_column('source', AtomicType('STRING'))])
self.catalog.alter_table('test.t$branch_dev', [
SchemaChange.update_column_type('count', AtomicType('BIGINT'))])
main = self.catalog.get_table('test.t')
dev = self.catalog.get_table('test.t$branch_dev')
assert main.table_schema.id == dev.table_schema.id
self._write(dev, {'id': 2, 'count': 3000000000})

switched = main.copy({'branch': 'dev'})
assert sorted(self._read(switched), key=lambda row: row['id']) == [
{'id': 1, 'count': 1}, {'id': 2, 'count': 3000000000}]
assert switched.copy({'branch': None}).fields == main.fields

self.catalog.alter_table('test.t$branch_dev', [
SchemaChange.add_column('quality', AtomicType('STRING'))])
latest = main.copy({'branch': 'dev'})
assert latest.field_names == ['id', 'count', 'quality']
assert self._read(latest.copy({'scan.tag-name': 'base'})) == [{'id': 1, 'count': 1}]
row = {'id': 3, 'count': 4000000000, 'quality': 'verified'}
self._write(latest, row)
assert row in self._read(self.catalog.get_table('test.t$branch_dev'))
assert self._read(main) == [{'id': 1, 'count': 1, 'source': None}]

def test_branch_copy_applies_dynamic_options_to_branch_defaults(self):
main = self._create_branched_table([DataField(0, 'id', AtomicType('INT'))], {'id': 1})
self.catalog.alter_table('test.t$branch_dev', [
SchemaChange.set_option('file.compression', 'zstd'),
SchemaChange.set_option('read.batch-size', '17'),
SchemaChange.set_option('read.parallelism', '3')])

configured = main.copy({'read.batch-size': '7', 'read.parallelism': None})
switched = configured.copy({'branch': 'dev'})
assert switched.table_schema.options['file.compression'] == 'zstd'
assert switched.table_schema.options['read.batch-size'] == '7'
assert 'read.parallelism' not in switched.table_schema.options
assert main.table_schema.options['file.compression'] == 'none'

@pytest.mark.parametrize('external', [False, True])
def test_branch_copy_loads_rest_branch_metadata(self, native_rest_catalog, external):
from pypaimon.common.file_io import FileIO
from pypaimon.schema.schema_manager import SchemaManager

catalog = native_rest_catalog
name = 'default.branch_copy'
fields = [DataField(0, 'id', AtomicType('INT'))]
options = {}
if external:
path = (self.root / 'warehouse' / 'default' / 'branch_copy').as_uri()
SchemaManager(FileIO.get(path), path).create_table(Schema(fields=fields))
options['path'] = path
catalog.create_table(name, Schema(fields=fields, options=options), False)
main = catalog.get_table(name)
assert main.copy({'branch': 'dev'}).current_branch() == 'dev'
catalog.create_branch(name, 'dev')
catalog.alter_table(name + '$branch_dev', [
SchemaChange.add_column('quality', AtomicType('STRING'))])

direct = catalog.get_table(name + '$branch_dev')
switched = main.copy({'branch': 'dev'})
assert switched.fields == direct.fields
assert switched.table_schema.options['path'] == direct.table_schema.options['path']
assert switched.file_io is main.file_io
assert switched.catalog_environment.identifier == direct.identifier
assert switched.catalog_environment.uuid is None
assert switched.catalog_environment.rest_table_response == main.catalog_environment.rest_table_response

def test_branch_copy_rejects_relocated_rest_table(self, native_rest_catalog):
catalog = native_rest_catalog
name = 'default.relocated'
fields = [DataField(0, 'id', AtomicType('INT'))]
catalog.create_table(name, Schema(fields=fields), False)
catalog.create_branch(name, 'dev')
dev = catalog.get_table(name + '$branch_dev')

catalog.drop_table(name)
location = (self.root / 'replacement').as_uri()
catalog.create_table(name, Schema(fields=fields, options={'path': location}), False)
assert catalog.get_table(name).table_path == location
with pytest.raises(ValueError, match='different table location'):
dev.copy({'branch': 'main'})

def test_commit_retry_refreshes_snapshot_schema(self, monkeypatch):
original = self._create([DataField(0, 'id', AtomicType('INT'))])
self._write(original, {'id': 1})
Expand Down
Loading