From d335897d0be020fc371cddc6f3043884a7e1f826 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Fri, 20 Nov 2026 00:40:12 -0800 Subject: [PATCH 1/2] [python] Export LeRobot v2 and v3 datasets from Paimon --- docs/docs/pypaimon/lerobot.md | 30 ++ .../pypaimon/multimodal/connection.py | 19 + .../pypaimon/multimodal/lerobot/__init__.py | 2 + .../pypaimon/multimodal/lerobot/export.py | 472 ++++++++++++++++++ .../pypaimon/tests/lerobot_export_test.py | 209 ++++++++ 5 files changed, 732 insertions(+) create mode 100644 paimon-python/pypaimon/multimodal/lerobot/export.py create mode 100644 paimon-python/pypaimon/tests/lerobot_export_test.py diff --git a/docs/docs/pypaimon/lerobot.md b/docs/docs/pypaimon/lerobot.md index 00f34e40dac3..c79d40cb69b1 100644 --- a/docs/docs/pypaimon/lerobot.md +++ b/docs/docs/pypaimon/lerobot.md @@ -402,6 +402,36 @@ By default, imports copy the original videos. Set re-encoding (requires PyAV 14.2+), retaining decoding dependencies. Each clip is limited to 64 MiB and 65,536 packets; unsupported timestamps fail the import. +## Export a LeRobot Dataset + +Export one source Dataset from a three-table group to a local LeRobot directory. +If the group contains multiple Datasets, select one by its `_datasets.id` UUID. +Pass the same `tag_name` used for training to export a fixed snapshot. +The destination directory must not exist. +Pass the table-group prefix, or the actual Frames table name for a group +created by `PaimonLeRobotWriter`. + +```python +conn.export_to_lerobot("robot_data", "/data/export-v3", version="v3.0") +conn.export_to_lerobot( + "robot_merged", "/data/export-v21", version="v2.1", + dataset_id="019a91e1-f035-79b9-b04b-44734dc2eb74", +) +# Use version="v2.0" for a LeRobot v2.0 consumer. +``` + +V3.0 keeps source-local indices and copies each encoded MP4 once, including +videos shared by multiple Episodes. V2.0 and v2.1 write one Parquet and one MP4 per +Episode and camera; RGB video export requires PyAV and re-encodes selected frames. +V2 export rejects depth video and video with audio. V2.0 requires source-level +statistics; v2.1 additionally writes per-Episode statistics. +Both exports write source Dataset metadata and Episode statistics. The output +can be opened with the matching LeRobot version. +`meta/paimon_source.json` preserves the Paimon Dataset ID, optional name, and +extra dataset text fields, which have no standard LeRobot metadata fields. +Nullable `subtask_index` values are preserved; older LeRobot readers that +reject null subtasks will also reject those source frames after export. + ## Capture LeRobot frames directly into Paimon `PaimonLeRobotWriter` implements the write-side surface used by LeRobot's diff --git a/paimon-python/pypaimon/multimodal/connection.py b/paimon-python/pypaimon/multimodal/connection.py index d47d13f2ca07..04dc05a89a93 100644 --- a/paimon-python/pypaimon/multimodal/connection.py +++ b/paimon-python/pypaimon/multimodal/connection.py @@ -193,6 +193,25 @@ def append_to_lerobot( append=True, ) + def export_to_lerobot( + self, + table_name: str, + output_dir, + *, + version: str = "v3.0", + dataset_id: Optional[str] = None, + tag_name: Optional[str] = None, + batch_size: int = 1024): + """Export one source Dataset from a LeRobot table group to a local directory. + + ``version`` is ``v3.0``, ``v2.1``, or ``v2.0``. Multi-source groups require a + ``dataset_id`` from ``_datasets.id``. + """ + from pypaimon.multimodal.lerobot.export import export_to_lerobot + return export_to_lerobot( + self, table_name, output_dir, version=version, + dataset_id=dataset_id, tag_name=tag_name, batch_size=batch_size) + def recover_lerobot_append(self, table_name: str): """Recover a three-table group after an interrupted append.""" from pypaimon.multimodal.lerobot.aggregate import _recover_incomplete_append diff --git a/paimon-python/pypaimon/multimodal/lerobot/__init__.py b/paimon-python/pypaimon/multimodal/lerobot/__init__.py index 482184a52b99..277cd1620dc2 100644 --- a/paimon-python/pypaimon/multimodal/lerobot/__init__.py +++ b/paimon-python/pypaimon/multimodal/lerobot/__init__.py @@ -17,6 +17,7 @@ """LeRobot Dataset v3 integration for multimodal Paimon tables.""" from pypaimon.multimodal.lerobot.api import load_from_lerobot +from pypaimon.multimodal.lerobot.export import export_to_lerobot from pypaimon.multimodal.lerobot.dataset import ( PaimonDatasetReader, PaimonLeRobotDataset, @@ -29,4 +30,5 @@ "PaimonLeRobotDataset", "PaimonLeRobotWriter", "load_from_lerobot", + "export_to_lerobot", ] diff --git a/paimon-python/pypaimon/multimodal/lerobot/export.py b/paimon-python/pypaimon/multimodal/lerobot/export.py new file mode 100644 index 000000000000..7c3dda6f19ea --- /dev/null +++ b/paimon-python/pypaimon/multimodal/lerobot/export.py @@ -0,0 +1,472 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Export a source Dataset from Paimon to LeRobot v2.0, v2.1, or v3.0.""" + +import copy +import hashlib +import json +import shutil +import tempfile +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +from pypaimon.catalog.catalog_exception import TableNotExistException +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.data.generic_variant import GenericVariant +from pypaimon.multimodal.lerobot.dataset import _component_table, _read_arrow +from pypaimon.multimodal.lerobot.metadata import ( + _companion_identifier, _companion_table_identifiers, _validate_tag_name, +) +from pypaimon.multimodal.lerobot.reader import _PaimonTableFrameReader +from pypaimon.multimodal.table import _target_schema +from pypaimon.table.row.blob import Blob, VideoFrameDescriptor + + +_V2_DATA_PATH = "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet" +_V2_VIDEO_PATH = "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4" +_V3_DATA_PATH = "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet" +_V3_VIDEO_PATH = "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4" + + +def export_to_lerobot(connection, table_name, output_dir, *, version="v3.0", + dataset_id=None, tag_name=None, batch_size=1024): + """Export one original Dataset; the destination must not already exist. + + The v3 exporter copies each encoded video payload once. The v2 exporter + writes one exact-frame MP4 per Episode and camera; this requires PyAV. + Both formats preserve source-local frame, Episode, and task indices. + """ + if version not in ("v2.0", "v2.1", "v3.0"): + raise ValueError("LeRobot export version must be v2.0, v2.1, or v3.0.") + if isinstance(batch_size, bool) or not isinstance(batch_size, int) \ + or batch_size <= 0: + raise ValueError("batch_size must be a positive integer.") + if dataset_id is not None and (not isinstance(dataset_id, str) + or not dataset_id): + raise ValueError("dataset_id must be a non-empty string.") + if tag_name is not None: + _validate_tag_name(tag_name) + target = Path(output_dir).expanduser().resolve() + if target.exists(): + raise FileExistsError(target) + + try: + frames = connection.get_table(table_name) + except TableNotExistException: + frames_name = _companion_identifier( + connection._identifier(table_name), "_frames") + frames = connection.get_table(frames_name) + raw_frames = frames.raw_table + identifiers = _companion_table_identifiers(raw_frames) + catalog = connection.catalog + read_frames = _component_table(catalog, raw_frames, tag_name) + datasets = _component_table( + catalog, catalog.get_table(identifiers["datasets"]), tag_name) + episodes = _component_table( + catalog, catalog.get_table(identifiers["episodes"]), tag_name) + sources = _read_arrow(datasets, _target_schema(datasets).names) + if dataset_id is None: + if sources.num_rows != 1: + raise ValueError( + "dataset_id is required when the LeRobot group has %d Datasets." + % sources.num_rows) + source = sources.to_pylist()[0] + else: + selected = [row for row in sources.to_pylist() + if row["id"] == dataset_id] + if len(selected) != 1: + raise ValueError("LeRobot dataset_id %r was not found." % dataset_id) + source = selected[0] + info = json.loads(source["info"]) + source_episodes = _read_source_episodes(episodes, source["id"]) + if len(source_episodes) != info["total_episodes"]: + raise ValueError("LeRobot Episode count disagrees with Dataset metadata.") + expected = 0 + for number, episode in enumerate(source_episodes): + if (episode["episode_index"] != number + or episode["dataset_from_index"] != expected + or episode["dataset_to_index"] - expected != episode["length"]): + raise ValueError("LeRobot source Episode ranges are not contiguous.") + expected = episode["dataset_to_index"] + if expected != info["total_frames"]: + raise ValueError("LeRobot Frame count disagrees with Dataset metadata.") + + target.parent.mkdir(parents=True, exist_ok=True) + stage = Path(tempfile.mkdtemp(prefix=".lerobot-export-", dir=target.parent)) + reader = None + try: + reader = _PaimonTableFrameReader( + read_frames, columns=_target_schema(read_frames).names) + _write_dataset(stage, reader, source, info, source_episodes, + version, batch_size) + reader.close() + reader = None + stage.rename(target) + return target + finally: + if reader is not None: + reader.close() + if stage.exists(): + shutil.rmtree(stage) + + +def _read_source_episodes(table, dataset_id): + builder = table.new_read_builder() + predicate = builder.new_predicate_builder().equal("dataset_id", dataset_id) + builder.with_filter(predicate) + plan = builder.new_scan().plan() + rows = builder.new_read().to_arrow(plan.splits()).to_pylist() + schema = _target_schema(table) + variant_groups = { + field.name for field in schema + if (field.metadata or {}).get(b"description") == + b"pypaimon.lerobot.episode-variant-group" + } + result = [] + for row in rows: + if row["dataset_id"] != dataset_id: + continue + original = {key: value for key, value in row.items() + if key not in ("dataset_id", "meta", "data", "videos", "stats") + and key not in variant_groups} + for group in ("meta", "data", "videos", "stats"): + if group not in schema.names: + continue + value = GenericVariant.from_arrow_struct(row[group]).to_python() + if group == "stats": + for feature, statistics in value.items(): + for name, item in statistics.items(): + original["stats/%s/%s" % (feature, name)] = item + else: + original.update({group + "/" + key: item + for key, item in value.items()}) + for group in variant_groups: + if row[group] is not None: + original.update({group + "/" + key: item + for key, item in GenericVariant.from_arrow_struct( + row[group]).to_python().items()}) + result.append(original) + return sorted(result, key=lambda row: row["episode_index"]) + + +def _write_dataset(stage, reader, source, info, episodes, version, batch_size): + video_keys = [name for name, feature in info["features"].items() + if feature.get("dtype") == "video"] + image_keys = [name for name, feature in info["features"].items() + if feature.get("dtype") == "image"] + frame_columns = list(info["features"]) + columns = ["dataset_id"] + frame_columns + physical_schema = _target_schema(reader._table) + frame_schema = pa.schema([ + pa.field(name, pa.struct([ + pa.field("bytes", pa.binary()), pa.field("path", pa.string())])) + if name in image_keys else physical_schema.field(name) + for name in frame_columns if name not in video_keys + ]) + (stage / "meta").mkdir() + source_identity = {key: value for key, value in source.items() + if key not in ("info", "tasks", "subtasks", "stats")} + (stage / "meta/paimon_source.json").write_text( + json.dumps(source_identity, ensure_ascii=False), encoding="utf-8") + tasks = sorted(source["tasks"], key=lambda row: row["task_index"]) + subtasks = source["subtasks"] + if version == "v3.0": + _write_v3_vocabulary(stage, "task", tasks) + if subtasks is not None: + _write_v3_vocabulary(stage, "subtask", subtasks) + else: + _write_jsonl(stage / "meta/tasks.jsonl", tasks) + if subtasks is not None: + _write_jsonl(stage / "meta/subtasks.jsonl", subtasks) + if source["stats"] is not None: + (stage / "meta/stats.json").write_text(source["stats"], encoding="utf-8") + + output_info = copy.deepcopy(info) + if version.startswith("v2."): + if version == "v2.0" and source["stats"] is None: + raise ValueError("LeRobot v2.0 export requires source-level statistics.") + for key in video_keys: + details = (info["features"][key].get("info") or + info["features"][key].get("video_info") or {}) + if details.get("video.is_depth_map") or details.get("has_audio") \ + or details.get("video.has_audio"): + raise ValueError( + "LeRobot v2 export supports RGB video without audio; " + "%s has depth or audio." % key) + output_info.update( + codebase_version=version, chunks_size=1000, + total_chunks=(len(episodes) + 999) // 1000, + total_videos=len(episodes) * len(video_keys), + data_path=_V2_DATA_PATH, + video_path=_V2_VIDEO_PATH if video_keys else None) + for key in video_keys: + feature = output_info["features"][key] + for name in ("info", "video_info"): + details = feature.get(name) + if not isinstance(details, dict): + continue + details.update({"video.codec": "h264", + "video.fps": int(info["fps"]), + "video.pix_fmt": "yuv420p", + "has_audio": False}) + for stale in ("video.gop_size", "video.bit_rate", + "video.extra_options", "video.video_backend"): + details.pop(stale, None) + if "video.has_audio" in details: + details["video.has_audio"] = False + if "video.encoder" in details: + details["video.encoder"] = "libx264" + if "video.crf" in details: + details["video.crf"] = 18 + else: + output_info.setdefault("robot_type", None) + output_info.setdefault("chunks_size", 1000) + output_info.setdefault("data_files_size_in_mb", 100) + output_info.setdefault("video_files_size_in_mb", 200) + output_info.setdefault("data_path", _V3_DATA_PATH) + output_info.setdefault("video_path", _V3_VIDEO_PATH if video_keys else None) + for episode in episodes: + episode.setdefault("data/chunk_index", episode["episode_index"] // 1000) + episode.setdefault("data/file_index", 0) + episode.setdefault("meta/episodes/chunk_index", episode["episode_index"] // 1000) + episode.setdefault("meta/episodes/file_index", 0) + parquet_writer = None + current_data_file = None + written_data_files = set() + v3_video_files = {} + episode_stats = [] + source_video_cache = stage / ".source-videos" + try: + for episode in episodes: + episode_index = episode["episode_index"] + begin, end = episode["dataset_from_index"], episode["dataset_to_index"] + if version.startswith("v2."): + data_path = stage / _V2_DATA_PATH.format( + episode_chunk=episode_index // 1000, + episode_index=episode_index) + else: + data_path = _safe_path(stage, output_info["data_path"].format( + chunk_index=episode["data/chunk_index"], + file_index=episode["data/file_index"])) + data_path.parent.mkdir(parents=True, exist_ok=True) + if data_path != current_data_file: + if parquet_writer is not None: + parquet_writer.close() + if data_path in written_data_files: + raise ValueError("LeRobot source data files are not Episode ordered.") + written_data_files.add(data_path) + current_data_file = data_path + parquet_writer = None + video_frames = {key: [] for key in video_keys} + for offset in range(begin, end, batch_size): + indices = list(range(offset, min(offset + batch_size, end))) + table = reader.read_keys( + [(source["id"], index) for index in indices], columns) + if table.num_rows != len(indices): + raise ValueError("LeRobot source Frame range is incomplete.") + table = table.sort_by("index") + rows = table.to_pylist() + if [row["index"] for row in rows] != indices \ + or any(row["dataset_id"] != source["id"] + or row["episode_index"] != episode_index + for row in rows): + raise ValueError("LeRobot source Frame keys are inconsistent.") + for row in rows: + del row["dataset_id"] + for key in video_keys: + video_frames[key].append( + VideoFrameDescriptor.deserialize(row.pop(key))) + for key in image_keys: + row[key] = {"bytes": _blob_bytes(reader.file_io, row[key]), + "path": None} + batch = pa.Table.from_pylist(rows, schema=frame_schema) + if parquet_writer is None: + parquet_writer = pq.ParquetWriter(data_path, batch.schema) + parquet_writer.write_table(batch) + + for key in video_keys: + descriptors = video_frames[key] + if not descriptors or any( + item.payload_descriptor != descriptors[0].payload_descriptor + for item in descriptors): + raise ValueError("LeRobot Episode video payload changes within an Episode.") + if version.startswith("v2."): + video_path = _safe_path(stage, _V2_VIDEO_PATH.format( + episode_chunk=episode_index // 1000, + episode_index=episode_index, video_key=key)) + payload = descriptors[0].payload_descriptor + cached = source_video_cache / hashlib.sha256( + (payload.uri + "\0" + str(payload.offset) + "\0" + + str(payload.length)).encode("utf-8")).hexdigest() + if not cached.exists(): + _copy_video(reader.file_io, payload, cached) + _encode_episode_video(cached, video_path, descriptors, + int(info["fps"])) + else: + payload = descriptors[0].payload_descriptor + identity = (payload.uri, payload.offset, payload.length) + key_files = v3_video_files.setdefault(key, {}) + if identity not in key_files: + chunk_index, file_index = divmod( + len(key_files), output_info["chunks_size"]) + video_path = _safe_path(stage, output_info["video_path"].format( + video_key=key, chunk_index=chunk_index, + file_index=file_index)) + _copy_video(reader.file_io, payload, video_path) + key_files[identity] = (chunk_index, file_index, video_path) + chunk_index, file_index, video_path = key_files[identity] + prefix = "videos/%s/" % key + episode[prefix + "chunk_index"] = chunk_index + episode[prefix + "file_index"] = file_index + if reader._table.options.options.get( + CoreOptions.WRITE_LEROBOT_REMUX_EPISODES): + start = _video_frame_time(video_path, descriptors[0].frame_index) + episode[prefix + "from_timestamp"] = start + episode[prefix + "to_timestamp"] = start + len(descriptors) / info["fps"] + if version.startswith("v2."): + episode_stats.append({"episode_index": episode_index, + "stats": _episode_stats(episode)}) + if parquet_writer is not None: + parquet_writer.close() + parquet_writer = None + if version == "v3.0": + _write_v3_episodes(stage, episodes) + if "total_videos" in output_info: + output_info["total_videos"] = sum( + len(files) for files in v3_video_files.values()) + else: + _write_jsonl(stage / "meta/episodes.jsonl", [ + {key: episode[key] for key in ("episode_index", "tasks", "length")} + for episode in episodes]) + if version == "v2.1": + _write_jsonl(stage / "meta/episodes_stats.jsonl", episode_stats) + (stage / "meta/info.json").write_text( + json.dumps(output_info, ensure_ascii=False), encoding="utf-8") + finally: + if parquet_writer is not None: + parquet_writer.close() + if source_video_cache.exists(): + shutil.rmtree(source_video_cache) + + +def _write_v3_vocabulary(stage, singular, rows): + import pandas as pd + + index = singular + "_index" + frame = pd.DataFrame(rows, columns=[index, singular]).set_index(singular) + pq.write_table(pa.Table.from_pandas(frame), + stage / ("meta/%ss.parquet" % singular)) + + +def _write_v3_episodes(stage, episodes): + for begin in range(0, len(episodes), 1000): + rows = [] + for episode in episodes[begin:begin + 1000]: + row = dict(episode) + if "meta/episodes/chunk_index" in row: + row["meta/episodes/chunk_index"] = begin // 1000 + if "meta/episodes/file_index" in row: + row["meta/episodes/file_index"] = 0 + rows.append(row) + names = list(dict.fromkeys(key for row in rows for key in row)) + path = stage / ("meta/episodes/chunk-%03d/file-000.parquet" % + (begin // 1000)) + path.parent.mkdir(parents=True, exist_ok=True) + pq.write_table(pa.Table.from_pylist([ + {key: row.get(key) for key in names} for row in rows]), path) + + +def _safe_path(root, relative): + path = (root / relative).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError("LeRobot metadata path escapes the output directory.") + return path + + +def _episode_stats(episode): + stats = {} + for key, value in episode.items(): + if key.startswith("stats/"): + feature, name = key[6:].rsplit("/", 1) + stats.setdefault(feature, {})[name] = value + return stats + + +def _write_jsonl(path, rows): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as stream: + for row in rows: + stream.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def _copy_video(file_io, descriptor, path): + path.parent.mkdir(parents=True, exist_ok=True) + reader = file_io.uri_reader_factory.create(descriptor.uri) + with Blob.from_descriptor(reader, descriptor).new_input_stream() as source, \ + path.open("wb") as target: + shutil.copyfileobj(source, target, length=1024 * 1024) + + +def _blob_bytes(file_io, value): + with Blob.from_bytes(value, file_io=file_io).new_input_stream() as source: + return source.read() + + +def _video_frame_time(path, index): + import av + + with av.open(str(path)) as container: + for ordinal, frame in enumerate(container.decode(video=0)): + if ordinal == index: + return float(frame.pts * frame.time_base) + raise ValueError("LeRobot video descriptor points beyond the MP4 frames.") + + +def _encode_episode_video(source_path, output_path, descriptors, fps): + try: + import av + except ImportError as error: + raise ImportError("LeRobot v2 video export requires PyAV.") from error + + wanted = [item.frame_index for item in descriptors] + if wanted != sorted(wanted): + raise ValueError("LeRobot Episode video frames are not ordered.") + output_path.parent.mkdir(parents=True, exist_ok=True) + position = 0 + with av.open(str(source_path)) as source, \ + av.open(str(output_path), mode="w", format="mp4") as target: + video = source.streams.video[0] + stream = target.add_stream("libx264", rate=fps) + stream.width, stream.height = video.width, video.height + stream.pix_fmt = "yuv420p" + stream.options = {"crf": "18", "threads": "4"} + for ordinal, frame in enumerate(source.decode(video=0)): + while position < len(wanted) and wanted[position] == ordinal: + encoded = av.VideoFrame.from_ndarray( + frame.to_ndarray(format="rgb24"), format="rgb24") + encoded.pts = position + for packet in stream.encode(encoded): + target.mux(packet) + position += 1 + if position == len(wanted): + break + if position != len(wanted): + raise ValueError("LeRobot video descriptor points beyond the MP4 frames.") + for packet in stream.encode(): + target.mux(packet) diff --git a/paimon-python/pypaimon/tests/lerobot_export_test.py b/paimon-python/pypaimon/tests/lerobot_export_test.py new file mode 100644 index 000000000000..8ebe223c154e --- /dev/null +++ b/paimon-python/pypaimon/tests/lerobot_export_test.py @@ -0,0 +1,209 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import io +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +import pypaimon.multimodal as pm +from pypaimon.multimodal.lerobot import PaimonLeRobotWriter +from pypaimon.tests.lerobot_aggregate_test import source + + +def _open_video_frames(path): + av = pytest.importorskip("av") + with av.open(str(path)) as container: + return len(list(container.decode(video=0))) + + +def test_export_v3_keeps_shared_mp4_and_reimports(tmp_path): + original = Path(source(tmp_path / "input", video=True)[7:]) + episode_path = original / "meta/episodes/chunk-000/file-000.parquet" + source_episodes = pq.read_table(episode_path).append_column( + "vendor/note", pa.array([None, "second episode"])) + pq.write_table(source_episodes, episode_path) + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + conn.load_from_lerobot("robot", original.as_uri()) + + output = conn.export_to_lerobot("robot", tmp_path / "v3") + assert output == tmp_path / "v3" + source_mp4 = original / "videos/camera/chunk-000/file-000.mp4" + exported_mp4 = output / "videos/camera/chunk-000/file-000.mp4" + assert exported_mp4.read_bytes() == source_mp4.read_bytes() + assert list(output.glob("videos/**/*.mp4")) == [exported_mp4] + assert pq.read_table(output / "data/chunk-000/file-000.parquet").num_rows == 5 + restored_episodes = pq.read_table( + output / "meta/episodes/chunk-000/file-000.parquet") + assert restored_episodes.num_rows == 2 + assert restored_episodes["vendor/note"].to_pylist() == [None, "second episode"] + + conn.load_from_lerobot("roundtrip", output.as_uri()) + assert conn.get_table("roundtrip_frames").scan().to_arrow().num_rows == 5 + + +@pytest.mark.parametrize("version", ["v2.0", "v2.1"]) +def test_export_v2_creates_one_exact_video_per_episode(tmp_path, version): + original = source(tmp_path / "input", video=True) + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + conn.load_from_lerobot("robot", original) + + output = conn.export_to_lerobot("robot", tmp_path / "v2", version=version) + info = json.loads((output / "meta/info.json").read_text()) + assert info["codebase_version"] == version + assert info["total_videos"] == 2 + assert (output / "meta/episodes_stats.jsonl").exists() == (version == "v2.1") + if version == "v2.1": + assert len((output / "meta/episodes_stats.jsonl").read_text().splitlines()) == 2 + assert len((output / "meta/episodes.jsonl").read_text().splitlines()) == 2 + assert _open_video_frames(output / "videos/chunk-000/camera/episode_000000.mp4") == 2 + assert _open_video_frames(output / "videos/chunk-000/camera/episode_000001.mp4") == 3 + assert pq.read_table(output / "data/chunk-000/episode_000000.parquet").num_rows == 2 + assert pq.read_table(output / "data/chunk-000/episode_000001.parquet").num_rows == 3 + + +def test_export_requires_dataset_id_for_multi_source_group(tmp_path): + first = source(tmp_path / "a") + second = source(tmp_path / "b", value=20) + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + conn.load_from_lerobot("robot", [first, second]) + with pytest.raises(ValueError, match="dataset_id is required"): + conn.export_to_lerobot("robot", tmp_path / "ambiguous") + ids = conn.get_table("robot_datasets").scan().to_arrow()["id"].to_pylist() + output = conn.export_to_lerobot("robot", tmp_path / "selected", dataset_id=ids[1]) + assert pq.read_table(output / "data/chunk-000/file-000.parquet")[ + "observation.state"][0].as_py()[0] == 20.0 + assert not (tmp_path / "ambiguous").exists() + + +def test_export_v3_remuxed_episode_videos(tmp_path): + original = Path(source(tmp_path / "input", video=True)[7:]) + info_path = original / "meta/info.json" + info = json.loads(info_path.read_text()) + info["chunks_size"] = 1 + info_path.write_text(json.dumps(info)) + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + conn.load_from_lerobot( + "robot", original.as_uri(), + options={"write.lerobot.remux-episodes": "true"}) + + output = conn.export_to_lerobot("robot", tmp_path / "v3") + assert sorted(path.relative_to(output).as_posix() + for path in output.glob("videos/**/*.mp4")) == [ + "videos/camera/chunk-000/file-000.mp4", + "videos/camera/chunk-001/file-000.mp4"] + episodes = pq.read_table(output / "meta/episodes/chunk-000/file-000.parquet") + assert episodes["videos/camera/chunk_index"].to_pylist() == [0, 1] + conn.load_from_lerobot("roundtrip", output.as_uri()) + assert conn.get_table("roundtrip_frames").scan().to_arrow().num_rows == 5 + + +def test_export_reads_one_tagged_group_snapshot(tmp_path): + first = source(tmp_path / "a") + second = source(tmp_path / "b", value=20) + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + conn.load_from_lerobot("robot", [first]) + conn.create_lerobot_tag("robot", "before-append") + conn.append_to_lerobot("robot", second) + + output = conn.export_to_lerobot( + "robot", tmp_path / "before", tag_name="before-append") + assert json.loads((output / "meta/info.json").read_text())["total_frames"] == 5 + assert pq.read_table(output / "data/chunk-000/file-000.parquet").num_rows == 5 + + +def test_export_image_feature_roundtrip(tmp_path): + image = pytest.importorskip("PIL.Image") + original = Path(source(tmp_path / "input")[7:]) + buffer = io.BytesIO() + image.new("RGB", (8, 8), (10, 20, 30)).save(buffer, format="PNG") + payload = buffer.getvalue() + info_path = original / "meta/info.json" + info = json.loads(info_path.read_text()) + info["features"]["camera.image"] = { + "dtype": "image", "shape": [8, 8, 3], + "names": ["height", "width", "channels"]} + info_path.write_text(json.dumps(info)) + data_path = original / "data/chunk-000/file-000.parquet" + table = pq.read_table(data_path) + table = table.append_column("camera.image", pa.array([ + {"bytes": payload, "path": None}] * table.num_rows)) + pq.write_table(table, data_path) + + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + conn.load_from_lerobot("robot", original.as_uri()) + output = conn.export_to_lerobot("robot", tmp_path / "v3") + images = pq.read_table(output / "data/chunk-000/file-000.parquet")[ + "camera.image"].to_pylist() + assert [value["bytes"] for value in images] == [payload] * 5 + conn.load_from_lerobot("roundtrip", output.as_uri()) + assert conn.get_table("roundtrip_frames").scan().to_arrow().num_rows == 5 + + +def test_export_direct_writer_group(tmp_path): + pytest.importorskip("lerobot.datasets.compute_stats") + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + writer = PaimonLeRobotWriter( + conn, "robot_frames", fps=10, + features={"action": {"dtype": "float32", "shape": (1,), + "names": None}}) + writer.add_frame({"action": np.array([1.0], dtype=np.float32), + "task": "pick"}) + writer.save_episode() + writer.finalize() + + for version in ("v3.0", "v2.1", "v2.0"): + output = conn.export_to_lerobot( + "robot_frames", tmp_path / version, version=version) + assert pq.read_table(next(output.glob("data/**/*.parquet"))).num_rows == 1 + if version == "v3.0": + from lerobot.datasets.lerobot_dataset import LeRobotDataset + + dataset = LeRobotDataset(repo_id="pypaimon/export-writer", root=output) + assert len(dataset) == 1 + assert dataset[0]["action"].item() == 1.0 + conn.load_from_lerobot("writer_roundtrip", output.as_uri()) + assert conn.get_table("writer_roundtrip_frames").scan().to_arrow().num_rows == 1 + + +def test_export_preserves_extra_dataset_text(tmp_path): + original = Path(source(tmp_path / "input")[7:]) + (original / "notes.txt").write_text("operator note\n", encoding="utf-8") + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + conn.load_from_lerobot( + "robot", original.as_uri(), dataset_name="source A", + extra_dataset_fields={"notes": "notes.txt"}) + + output = conn.export_to_lerobot("robot", tmp_path / "v3") + identity = json.loads((output / "meta/paimon_source.json").read_text()) + assert identity["name"] == "source A" + assert identity["notes"] == "operator note\n" + assert identity["id"] == conn.get_table("robot_datasets").scan().to_arrow()[ + "id"][0].as_py() + + +def test_export_v20_requires_source_statistics(tmp_path): + original = Path(source(tmp_path / "input")[7:]) + (original / "meta/stats.json").unlink() + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + conn.load_from_lerobot("robot", original.as_uri()) + with pytest.raises(ValueError, match="requires source-level statistics"): + conn.export_to_lerobot("robot", tmp_path / "v20", version="v2.0") + assert not (tmp_path / "v20").exists() From e50262571fd0b4677a12500101cb77cc67c499cb Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Fri, 20 Nov 2026 01:18:18 -0800 Subject: [PATCH 2/2] Support exporting LeRobot datasets to OSS --- docs/docs/pypaimon/lerobot.md | 14 +++- .../pypaimon/multimodal/connection.py | 8 ++- .../pypaimon/multimodal/lerobot/export.py | 72 +++++++++++++++++-- .../pypaimon/tests/lerobot_export_test.py | 53 ++++++++++++++ 4 files changed, 135 insertions(+), 12 deletions(-) diff --git a/docs/docs/pypaimon/lerobot.md b/docs/docs/pypaimon/lerobot.md index c79d40cb69b1..cc4e461959ea 100644 --- a/docs/docs/pypaimon/lerobot.md +++ b/docs/docs/pypaimon/lerobot.md @@ -404,10 +404,11 @@ to 64 MiB and 65,536 packets; unsupported timestamps fail the import. ## Export a LeRobot Dataset -Export one source Dataset from a three-table group to a local LeRobot directory. +Export one source Dataset from a three-table group to a local directory or an +`oss://bucket/prefix` LeRobot dataset. If the group contains multiple Datasets, select one by its `_datasets.id` UUID. Pass the same `tag_name` used for training to export a fixed snapshot. -The destination directory must not exist. +The destination directory or OSS prefix must not exist. Pass the table-group prefix, or the actual Frames table name for a group created by `PaimonLeRobotWriter`. @@ -418,8 +419,17 @@ conn.export_to_lerobot( dataset_id="019a91e1-f035-79b9-b04b-44734dc2eb74", ) # Use version="v2.0" for a LeRobot v2.0 consumer. +conn.export_to_lerobot("robot_data", "oss://my-bucket/lerobot/export-v3") ``` +OSS output uses the table's FileIO when it can access the target bucket. Pass +`output_options` with `fs.oss.endpoint`, `fs.oss.accessKeyId`, +`fs.oss.accessKeySecret`, and optional `fs.oss.securityToken` when the output +bucket has separate credentials. The exporter stages files in the worker's +temporary directory, uploads them to OSS, and writes `meta/info.json` last. +An incomplete upload leaves its prefix for inspection; choose a new prefix or +remove it before retrying. + V3.0 keeps source-local indices and copies each encoded MP4 once, including videos shared by multiple Episodes. V2.0 and v2.1 write one Parquet and one MP4 per Episode and camera; RGB video export requires PyAV and re-encodes selected frames. diff --git a/paimon-python/pypaimon/multimodal/connection.py b/paimon-python/pypaimon/multimodal/connection.py index 04dc05a89a93..2f0c8a4357f4 100644 --- a/paimon-python/pypaimon/multimodal/connection.py +++ b/paimon-python/pypaimon/multimodal/connection.py @@ -201,8 +201,9 @@ def export_to_lerobot( version: str = "v3.0", dataset_id: Optional[str] = None, tag_name: Optional[str] = None, - batch_size: int = 1024): - """Export one source Dataset from a LeRobot table group to a local directory. + batch_size: int = 1024, + output_options: Optional[Dict[str, str]] = None): + """Export one source Dataset to a local directory or OSS prefix. ``version`` is ``v3.0``, ``v2.1``, or ``v2.0``. Multi-source groups require a ``dataset_id`` from ``_datasets.id``. @@ -210,7 +211,8 @@ def export_to_lerobot( from pypaimon.multimodal.lerobot.export import export_to_lerobot return export_to_lerobot( self, table_name, output_dir, version=version, - dataset_id=dataset_id, tag_name=tag_name, batch_size=batch_size) + dataset_id=dataset_id, tag_name=tag_name, batch_size=batch_size, + output_options=output_options) def recover_lerobot_append(self, table_name: str): """Recover a three-table group after an interrupted append.""" diff --git a/paimon-python/pypaimon/multimodal/lerobot/export.py b/paimon-python/pypaimon/multimodal/lerobot/export.py index 7c3dda6f19ea..eead334b27e8 100644 --- a/paimon-python/pypaimon/multimodal/lerobot/export.py +++ b/paimon-python/pypaimon/multimodal/lerobot/export.py @@ -19,15 +19,19 @@ import copy import hashlib import json +import os import shutil import tempfile +from collections.abc import Mapping from pathlib import Path +from urllib.parse import urlparse import pyarrow as pa import pyarrow.parquet as pq from pypaimon.catalog.catalog_exception import TableNotExistException from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options import Options from pypaimon.data.generic_variant import GenericVariant from pypaimon.multimodal.lerobot.dataset import _component_table, _read_arrow from pypaimon.multimodal.lerobot.metadata import ( @@ -35,6 +39,7 @@ ) from pypaimon.multimodal.lerobot.reader import _PaimonTableFrameReader from pypaimon.multimodal.table import _target_schema +from pypaimon.common.file_io import FileIO from pypaimon.table.row.blob import Blob, VideoFrameDescriptor @@ -45,7 +50,8 @@ def export_to_lerobot(connection, table_name, output_dir, *, version="v3.0", - dataset_id=None, tag_name=None, batch_size=1024): + dataset_id=None, tag_name=None, batch_size=1024, + output_options=None): """Export one original Dataset; the destination must not already exist. The v3 exporter copies each encoded video payload once. The v2 exporter @@ -62,9 +68,28 @@ def export_to_lerobot(connection, table_name, output_dir, *, version="v3.0", raise ValueError("dataset_id must be a non-empty string.") if tag_name is not None: _validate_tag_name(tag_name) - target = Path(output_dir).expanduser().resolve() - if target.exists(): - raise FileExistsError(target) + location = os.fspath(output_dir) + if not isinstance(location, str): + raise ValueError("output_dir must be a local path or oss:// URI.") + parsed = urlparse(location) + if parsed.scheme not in ("", "oss"): + raise ValueError("output_dir must be a local path or oss:// URI.") + remote = parsed.scheme == "oss" + if remote: + parent_segment = any(part in (".", "..") + for part in parsed.path.split("/")) + if (not parsed.netloc or not parsed.path.strip("/") or parsed.query + or parsed.fragment or parent_segment): + raise ValueError("output_dir must be an OSS bucket and a nonempty prefix.") + target_uri = location.rstrip("/") + if output_options is not None and not isinstance(output_options, Mapping): + raise ValueError("output_options must be a mapping.") + else: + if output_options is not None: + raise ValueError("output_options is only used with OSS output.") + target = Path(location).expanduser().resolve() + if target.exists(): + raise FileExistsError(target) try: frames = connection.get_table(table_name) @@ -107,23 +132,56 @@ def export_to_lerobot(connection, table_name, output_dir, *, version="v3.0", if expected != info["total_frames"]: raise ValueError("LeRobot Frame count disagrees with Dataset metadata.") - target.parent.mkdir(parents=True, exist_ok=True) - stage = Path(tempfile.mkdtemp(prefix=".lerobot-export-", dir=target.parent)) + stage = None reader = None + output_io = None try: + if remote: + output_io = (FileIO.get(target_uri, Options(dict(output_options))) + if output_options is not None else read_frames.file_io) + if output_io.exists(target_uri): + raise FileExistsError(target_uri) + else: + target.parent.mkdir(parents=True, exist_ok=True) + stage = Path(tempfile.mkdtemp( + prefix=".lerobot-export-", dir=None if remote else target.parent)) reader = _PaimonTableFrameReader( read_frames, columns=_target_schema(read_frames).names) _write_dataset(stage, reader, source, info, source_episodes, version, batch_size) reader.close() reader = None + if remote: + _upload_to_oss(stage, target_uri, output_io) + return target_uri stage.rename(target) return target finally: if reader is not None: reader.close() - if stage.exists(): + if stage is not None and stage.exists(): shutil.rmtree(stage) + if output_options is not None and output_io is not None: + output_io.close() + + +def _upload_to_oss(stage, prefix, file_io): + if file_io.exists(prefix): + raise FileExistsError(prefix) + info = stage / "meta/info.json" + for path in sorted(stage.rglob("*")): + if not path.is_file() or path == info: + continue + _upload_file(path, prefix + "/" + path.relative_to(stage).as_posix(), + file_io) + _upload_file(info, prefix + "/meta/info.json", file_io) + + +def _upload_file(path, uri, file_io): + if file_io.exists(uri): + raise FileExistsError(uri) + with path.open("rb") as source, file_io.new_output_stream(uri) as target: + shutil.copyfileobj(source, target, length=1024 * 1024) def _read_source_episodes(table, dataset_id): diff --git a/paimon-python/pypaimon/tests/lerobot_export_test.py b/paimon-python/pypaimon/tests/lerobot_export_test.py index 8ebe223c154e..327dda13970a 100644 --- a/paimon-python/pypaimon/tests/lerobot_export_test.py +++ b/paimon-python/pypaimon/tests/lerobot_export_test.py @@ -17,6 +17,7 @@ import json import io from pathlib import Path +from unittest import mock import numpy as np import pyarrow as pa @@ -24,6 +25,7 @@ import pytest import pypaimon.multimodal as pm +from pypaimon.common.file_io import FileIO from pypaimon.multimodal.lerobot import PaimonLeRobotWriter from pypaimon.tests.lerobot_aggregate_test import source @@ -207,3 +209,54 @@ def test_export_v20_requires_source_statistics(tmp_path): with pytest.raises(ValueError, match="requires source-level statistics"): conn.export_to_lerobot("robot", tmp_path / "v20", version="v2.0") assert not (tmp_path / "v20").exists() + + +@pytest.mark.parametrize("version", ["v3.0", "v2.1"]) +def test_export_to_oss_prefix(tmp_path, version): + class MemoryOss: + def __init__(self): + self.objects = {} + self.writes = [] + + def exists(self, uri): + return uri in self.objects or any( + key.startswith(uri.rstrip("/") + "/") for key in self.objects) + + def new_output_stream(self, uri): + store = self + + class Output(io.BytesIO): + def close(self): + store.objects[uri] = self.getvalue() + store.writes.append(uri) + super().close() + + return Output() + + def close(self): + pass + + original = source(tmp_path / "input", video=True) + conn = pm.connect(options={"warehouse": str(tmp_path / "warehouse")}) + conn.load_from_lerobot("robot", original) + target = MemoryOss() + prefix = "oss://test-bucket/lerobot/export" + original_get = FileIO.get + with mock.patch("pypaimon.multimodal.lerobot.export.FileIO.get", + side_effect=lambda uri, options=None: + target if uri.startswith("oss://") else original_get(uri, options)): + assert conn.export_to_lerobot( + "robot", prefix, version=version, + output_options={"fs.oss.endpoint": "test-endpoint"}) == prefix + assert target.writes[-1] == prefix + "/meta/info.json" + assert len([key for key in target.objects if key.endswith(".mp4")]) == ( + 1 if version == "v3.0" else 2) + data_file = next(value for key, value in target.objects.items() + if key.endswith(".parquet") and "/data/" in key) + assert pq.read_table(io.BytesIO(data_file)).num_rows > 0 + before = len(target.writes) + with pytest.raises(FileExistsError): + conn.export_to_lerobot( + "robot", prefix, version=version, + output_options={"fs.oss.endpoint": "test-endpoint"}) + assert len(target.writes) == before